Apache Solr open-source search software
LUCENE-9570: code reformatting [partial].
Dawid Weiss6faa4f98e048
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradleindex 87fcdb6bdeb..8a06eefe2ff 100644--- a/gradle/validation/spotless.gradle+++ b/gradle/validation/spotless.gradle@@ -33,7 +33,26 @@ allprojects { prj ->googleJavaFormat('1.9')switch (project.path) {- // Disable for everything else for now.+ // These modules are complete - all sources scanned.+ case ":lucene:highlighter":+ target "src/**"+ targetExclude "**/resources/**", "**/CambridgeMA.utf8", "**/overview.html"+ break++ // Partially complete.+ case ":lucene:core":+ target "src/**/org/apache/lucene/analysis/standard/**",+ "src/**/org/apache/lucene/analysis/tokenattributes/**"+ targetExclude "**/resources/**", "**/StandardTokenizerImpl.jflex", "**/StandardTokenizerImpl.java"+ break++ case ":lucene:analysis:common":+ target "src/**/org/apache/lucene/analysis/ar/**",+ "src/**/org/apache/lucene/collation/**"+ targetExclude "**/resources/**"+ break++ // All others - disable reformatting/ checks for now.default:target 'non-existing/**'breakdiff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.javaindex 90a60a5f5cd..ce679b11680 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java@@ -16,10 +16,8 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;import java.io.Reader;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CharArraySet;import org.apache.lucene.analysis.LowerCaseFilter;@@ -32,19 +30,17 @@ import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;import org.apache.lucene.analysis.standard.StandardTokenizer;/**- * {@link Analyzer} for Arabic.- * <p>- * This analyzer implements light-stemming as specified by:- * <i>- * Light Stemming for Arabic Information Retrieval- * </i>- * http://www.mtholyoke.edu/~lballest/Pubs/arab_stem05.pdf- * <p>- * The analysis package contains three primary components:+ * {@link Analyzer} for Arabic.+ *+ * <p>This analyzer implements light-stemming as specified by: <i> Light Stemming for Arabic+ * Information Retrieval </i> http://www.mtholyoke.edu/~lballest/Pubs/arab_stem05.pdf+ *+ * <p>The analysis package contains three primary components:+ ** <ul>- * <li>{@link ArabicNormalizationFilter}: Arabic orthographic normalization.- * <li>{@link ArabicStemFilter}: Arabic light stemming- * <li>Arabic stop words file: a set of default Arabic stop words.+ * <li>{@link ArabicNormalizationFilter}: Arabic orthographic normalization.+ * <li>{@link ArabicStemFilter}: Arabic light stemming+ * <li>Arabic stop words file: a set of default Arabic stop words.* </ul>** @since 3.1@@ -53,23 +49,24 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {/*** File containing default Arabic stopwords.- *- * Default stopword list is from http://members.unine.ch/jacques.savoy/clef/index.html- * The stopword list is BSD-Licensed.+ *+ * <p>Default stopword list is from http://members.unine.ch/jacques.savoy/clef/index.html The+ * stopword list is BSD-Licensed.*/- public final static String DEFAULT_STOPWORD_FILE = "stopwords.txt";+ public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";/*** Returns an unmodifiable instance of the default stop-words set.+ ** @return an unmodifiable instance of the default stop-words set.*/- public static CharArraySet getDefaultStopSet(){+ public static CharArraySet getDefaultStopSet() {return DefaultSetHolder.DEFAULT_STOP_SET;}-+/**- * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class- * accesses the static final set the first time.;+ * Atomically loads the DEFAULT_STOP_SET in a lazy fashion once the outer class accesses the+ * static final set the first time.;*/private static class DefaultSetHolder {static final CharArraySet DEFAULT_STOP_SET;@@ -84,51 +81,43 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {}}}-+private final CharArraySet stemExclusionSet;- /**- * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.- */+ /** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */public ArabicAnalyzer() {this(DefaultSetHolder.DEFAULT_STOP_SET);}-+/*** Builds an analyzer with the given stop words- *- * @param stopwords- * a stopword set+ *+ * @param stopwords a stopword set*/- public ArabicAnalyzer(CharArraySet stopwords){+ public ArabicAnalyzer(CharArraySet stopwords) {this(stopwords, CharArraySet.EMPTY_SET);}/**- * Builds an analyzer with the given stop word. If a none-empty stem exclusion set is- * provided this analyzer will add a {@link SetKeywordMarkerFilter} before- * {@link ArabicStemFilter}.- *- * @param stopwords- * a stopword set- * @param stemExclusionSet- * a set of terms not to be stemmed+ * Builds an analyzer with the given stop word. If a none-empty stem exclusion set is provided+ * this analyzer will add a {@link SetKeywordMarkerFilter} before {@link ArabicStemFilter}.+ *+ * @param stopwords a stopword set+ * @param stemExclusionSet a set of terms not to be stemmed*/- public ArabicAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet){+ public ArabicAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) {super(stopwords);this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet));}/**- * Creates- * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}- * used to tokenize all the text in the provided {@link Reader}.- *- * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}- * built from an {@link StandardTokenizer} filtered with- * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link StopFilter},- * {@link ArabicNormalizationFilter}, {@link SetKeywordMarkerFilter}- * if a stem exclusion set is provided and {@link ArabicStemFilter}.+ * Creates {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} used to tokenize all+ * the text in the provided {@link Reader}.+ *+ * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from an {@link+ * StandardTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter},+ * {@link StopFilter}, {@link ArabicNormalizationFilter}, {@link SetKeywordMarkerFilter} if a+ * stem exclusion set is provided and {@link ArabicStemFilter}.*/@Overrideprotected TokenStreamComponents createComponents(String fieldName) {@@ -139,7 +128,7 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {result = new StopFilter(result, stopwords);// TODO maybe we should make ArabicNormalization filter also KeywordAttribute aware?!result = new ArabicNormalizationFilter(result);- if(!stemExclusionSet.isEmpty()) {+ if (!stemExclusionSet.isEmpty()) {result = new SetKeywordMarkerFilter(result, stemExclusionSet);}return new TokenStreamComponents(source, new ArabicStemFilter(result));@@ -153,4 +142,3 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {return result;}}-diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.javaindex ba941061c26..8ac1b0deadc 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java@@ -16,22 +16,16 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;-import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;-/**- * A {@link TokenFilter} that applies {@link ArabicNormalizer} to normalize the orthography.- *- */-+/** A {@link TokenFilter} that applies {@link ArabicNormalizer} to normalize the orthography. */public final class ArabicNormalizationFilter extends TokenFilter {private final ArabicNormalizer normalizer = new ArabicNormalizer();private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);-+public ArabicNormalizationFilter(TokenStream input) {super(input);}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.javaindex 5f604f58a00..937752e173e 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java@@ -16,14 +16,13 @@*/package org.apache.lucene.analysis.ar;-import java.util.Map;--import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.TokenFilterFactory;+import org.apache.lucene.analysis.TokenStream;/*** Factory for {@link ArabicNormalizationFilter}.+ ** <pre class="prettyprint">* <fieldType name="text_arnormal" class="solr.TextField" positionIncrementGap="100">* <analyzer>@@ -41,7 +40,7 @@ public class ArabicNormalizationFilterFactory extends TokenFilterFactory {public static final String NAME = "arabicNormalization";/** Creates a new ArabicNormalizationFilterFactory */- public ArabicNormalizationFilterFactory(Map<String,String> args) {+ public ArabicNormalizationFilterFactory(Map<String, String> args) {super(args);if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameters: " + args);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizer.javaindex 92a818d95d0..bea82f6e5d4 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizer.java@@ -16,23 +16,22 @@*/package org.apache.lucene.analysis.ar;-import static org.apache.lucene.analysis.util.StemmerUtil.*;/**- * Normalizer for Arabic.- * <p>- * Normalization is done in-place for efficiency, operating on a termbuffer.- * <p>- * Normalization is defined as:- * <ul>- * <li> Normalization of hamza with alef seat to a bare alef.- * <li> Normalization of teh marbuta to heh- * <li> Normalization of dotless yeh (alef maksura) to yeh.- * <li> Removal of Arabic diacritics (the harakat)- * <li> Removal of tatweel (stretching character).- * </ul>+ * Normalizer for Arabic.+ *+ * <p>Normalization is done in-place for efficiency, operating on a termbuffer.*+ * <p>Normalization is defined as:+ *+ * <ul>+ * <li>Normalization of hamza with alef seat to a bare alef.+ * <li>Normalization of teh marbuta to heh+ * <li>Normalization of dotless yeh (alef maksura) to yeh.+ * <li>Removal of Arabic diacritics (the harakat)+ * <li>Removal of tatweel (stretching character).+ * </ul>*/public class ArabicNormalizer {public static final char ALEF = '\u0627';@@ -59,7 +58,7 @@ public class ArabicNormalizer {/*** Normalize an input buffer of Arabic text- *+ ** @param s input buffer* @param len length of input buffer* @return length of input buffer after normalization@@ -68,31 +67,31 @@ public class ArabicNormalizer {for (int i = 0; i < len; i++) {switch (s[i]) {- case ALEF_MADDA:- case ALEF_HAMZA_ABOVE:- case ALEF_HAMZA_BELOW:- s[i] = ALEF;- break;- case DOTLESS_YEH:- s[i] = YEH;- break;- case TEH_MARBUTA:- s[i] = HEH;- break;- case TATWEEL:- case KASRATAN:- case DAMMATAN:- case FATHATAN:- case FATHA:- case DAMMA:- case KASRA:- case SHADDA:- case SUKUN:- len = delete(s, i, len);- i--;- break;- default:- break;+ case ALEF_MADDA:+ case ALEF_HAMZA_ABOVE:+ case ALEF_HAMZA_BELOW:+ s[i] = ALEF;+ break;+ case DOTLESS_YEH:+ s[i] = YEH;+ break;+ case TEH_MARBUTA:+ s[i] = HEH;+ break;+ case TATWEEL:+ case KASRATAN:+ case DAMMATAN:+ case FATHATAN:+ case FATHA:+ case DAMMA:+ case KASRA:+ case SHADDA:+ case SUKUN:+ len = delete(s, i, len);+ i--;+ break;+ default:+ break;}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.javaindex f1d0fa3c164..9005ea9ccb1 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java@@ -16,29 +16,27 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;--import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // javadoc @linkimport org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;-import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; // javadoc @linkimport org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;/*** A {@link TokenFilter} that applies {@link ArabicStemmer} to stem Arabic words..- * <p>- * To prevent terms from being stemmed use an instance of- * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets- * the {@link KeywordAttribute} before this {@link TokenStream}.- * </p>- * @see SetKeywordMarkerFilter */-+ *+ * <p>To prevent terms from being stemmed use an instance of {@link SetKeywordMarkerFilter} or a+ * custom {@link TokenFilter} that sets the {@link KeywordAttribute} before this {@link+ * TokenStream}.+ *+ * @see SetKeywordMarkerFilter+ */public final class ArabicStemFilter extends TokenFilter {private final ArabicStemmer stemmer = new ArabicStemmer();private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);-+public ArabicStemFilter(TokenStream input) {super(input);}@@ -46,7 +44,7 @@ public final class ArabicStemFilter extends TokenFilter {@Overridepublic boolean incrementToken() throws IOException {if (input.incrementToken()) {- if(!keywordAttr.isKeyword()) {+ if (!keywordAttr.isKeyword()) {final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());termAtt.setLength(newlen);}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilterFactory.javaindex 96c225afee1..9f2e70270c7 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilterFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilterFactory.java@@ -16,14 +16,13 @@*/package org.apache.lucene.analysis.ar;-import java.util.Map;--import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.TokenFilterFactory;+import org.apache.lucene.analysis.TokenStream;/*** Factory for {@link ArabicStemFilter}.+ ** <pre class="prettyprint">* <fieldType name="text_arstem" class="solr.TextField" positionIncrementGap="100">* <analyzer>@@ -42,7 +41,7 @@ public class ArabicStemFilterFactory extends TokenFilterFactory {public static final String NAME = "arabicStem";/** Creates a new ArabicStemFilterFactory */- public ArabicStemFilterFactory(Map<String,String> args) {+ public ArabicStemFilterFactory(Map<String, String> args) {super(args);if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameters: " + args);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.javaindex 6093317d3b0..5675bcf53f9 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java@@ -16,21 +16,19 @@*/package org.apache.lucene.analysis.ar;--import static org.apache.lucene.analysis.util.StemmerUtil.*;/**- * Stemmer for Arabic.- * <p>- * Stemming is done in-place for efficiency, operating on a termbuffer.- * <p>- * Stemming is defined as:- * <ul>- * <li> Removal of attached definite article, conjunction, and prepositions.- * <li> Stemming of common suffixes.- * </ul>+ * Stemmer for Arabic.+ *+ * <p>Stemming is done in-place for efficiency, operating on a termbuffer.+ *+ * <p>Stemming is defined as:*+ * <ul>+ * <li>Removal of attached definite article, conjunction, and prepositions.+ * <li>Stemming of common suffixes.+ * </ul>*/public class ArabicStemmer {public static final char ALEF = '\u0627';@@ -44,33 +42,33 @@ public class ArabicStemmer {public static final char HEH = '\u0647';public static final char WAW = '\u0648';public static final char YEH = '\u064A';-+public static final char prefixes[][] = {- ("" + ALEF + LAM).toCharArray(),- ("" + WAW + ALEF + LAM).toCharArray(),- ("" + BEH + ALEF + LAM).toCharArray(),- ("" + KAF + ALEF + LAM).toCharArray(),- ("" + FEH + ALEF + LAM).toCharArray(),- ("" + LAM + LAM).toCharArray(),- ("" + WAW).toCharArray(),+ ("" + ALEF + LAM).toCharArray(),+ ("" + WAW + ALEF + LAM).toCharArray(),+ ("" + BEH + ALEF + LAM).toCharArray(),+ ("" + KAF + ALEF + LAM).toCharArray(),+ ("" + FEH + ALEF + LAM).toCharArray(),+ ("" + LAM + LAM).toCharArray(),+ ("" + WAW).toCharArray(),};-+public static final char suffixes[][] = {- ("" + HEH + ALEF).toCharArray(),- ("" + ALEF + NOON).toCharArray(),- ("" + ALEF + TEH).toCharArray(),- ("" + WAW + NOON).toCharArray(),- ("" + YEH + NOON).toCharArray(),+ ("" + HEH + ALEF).toCharArray(),+ ("" + ALEF + NOON).toCharArray(),+ ("" + ALEF + TEH).toCharArray(),+ ("" + WAW + NOON).toCharArray(),+ ("" + YEH + NOON).toCharArray(),("" + YEH + HEH).toCharArray(),("" + YEH + TEH_MARBUTA).toCharArray(),("" + HEH).toCharArray(),("" + TEH_MARBUTA).toCharArray(),("" + YEH).toCharArray(),-};-+ };+/*** Stem an input buffer of Arabic text.- *+ ** @param s input buffer* @param len length of input buffer* @return length of input buffer after normalization@@ -78,38 +76,40 @@ public class ArabicStemmer {public int stem(char s[], int len) {len = stemPrefix(s, len);len = stemSuffix(s, len);-+return len;}-+/*** Stem a prefix off an Arabic word.+ ** @param s input buffer* @param len length of input buffer* @return new length of input buffer after stemming.*/public int stemPrefix(char s[], int len) {- for (int i = 0; i < prefixes.length; i++)- if (startsWithCheckLength(s, len, prefixes[i]))- return deleteN(s, 0, len, prefixes[i].length);+ for (int i = 0; i < prefixes.length; i++)+ if (startsWithCheckLength(s, len, prefixes[i])) return deleteN(s, 0, len, prefixes[i].length);return len;}/*** Stem suffix(es) off an Arabic word.+ ** @param s input buffer* @param len length of input buffer* @return new length of input buffer after stemming*/public int stemSuffix(char s[], int len) {- for (int i = 0; i < suffixes.length; i++)+ for (int i = 0; i < suffixes.length; i++)if (endsWithCheckLength(s, len, suffixes[i]))len = deleteN(s, len - suffixes[i].length, len, suffixes[i].length);return len;}-+/*** Returns true if the prefix matches and can be stemmed+ ** @param s input buffer* @param len length of input buffer* @param prefix prefix to check@@ -121,16 +121,19 @@ public class ArabicStemmer {} else if (len < prefix.length + 2) { // other prefixes require only 2.return false;} else {- for (int i = 0; i < prefix.length; i++)- if (s[i] != prefix[i])+ for (int i = 0; i < prefix.length; i++) {+ if (s[i] != prefix[i]) {return false;-+ }+ }+return true;}}-+/*** Returns true if the suffix matches and can be stemmed+ ** @param s input buffer* @param len length of input buffer* @param suffix suffix to check@@ -140,11 +143,13 @@ public class ArabicStemmer {if (len < suffix.length + 2) { // all suffixes require at least 2 characters after stemmingreturn false;} else {- for (int i = 0; i < suffix.length; i++)- if (s[len - suffix.length + i] != suffix[i])+ for (int i = 0; i < suffix.length; i++) {+ if (s[len - suffix.length + i] != suffix[i]) {return false;-+ }+ }+return true;}- }+ }}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/package-info.javaindex dfb2cab2dbd..3d2148be27f 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/package-info.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/package-info.java@@ -15,7 +15,5 @@* limitations under the License.*/-/**- * Analyzer for Arabic.- */-package org.apache.lucene.analysis.ar;\ No newline at end of file+/** Analyzer for Arabic. */+package org.apache.lucene.analysis.ar;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationAttributeFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationAttributeFactory.javaindex 6de59c8ccea..3058b68edcb 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationAttributeFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationAttributeFactory.java@@ -16,73 +16,59 @@*/package org.apache.lucene.collation;-import java.text.Collator;-import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.collation.tokenattributes.CollatedTermAttributeImpl;import org.apache.lucene.util.AttributeFactory;/**- * <p>- * Converts each token into its {@link java.text.CollationKey}, and then- * encodes the bytes as an index term.- * </p>- * <p>- * <strong>WARNING:</strong> Make sure you use exactly the same Collator at- * index and query time -- CollationKeys are only comparable when produced by- * the same Collator. Since {@link java.text.RuleBasedCollator}s are not- * independently versioned, it is unsafe to search against stored- * CollationKeys unless the following are exactly the same (best practice is- * to store this information with the index and check that they remain the- * same at query time):- * </p>+ * Converts each token into its {@link java.text.CollationKey}, and then encodes the bytes as an+ * index term.+ *+ * <p><strong>WARNING:</strong> Make sure you use exactly the same Collator at index and query time+ * -- CollationKeys are only comparable when produced by the same Collator. Since {@link+ * java.text.RuleBasedCollator}s are not independently versioned, it is unsafe to search against+ * stored CollationKeys unless the following are exactly the same (best practice is to store this+ * information with the index and check that they remain the same at query time):+ ** <ol>- * <li>JVM vendor</li>- * <li>JVM version, including patch version</li>- * <li>- * The language (and country and variant, if specified) of the Locale- * used when constructing the collator via- * {@link Collator#getInstance(java.util.Locale)}.- * </li>- * <li>- * The collation strength used - see {@link Collator#setStrength(int)}- * </li>- * </ol>- * <p>- * The <code>ICUCollationAttributeFactory</code> in the analysis-icu package- * uses ICU4J's Collator, which makes its- * version available, thus allowing collation to be versioned independently- * from the JVM. ICUCollationAttributeFactory is also significantly faster and- * generates significantly shorter keys than CollationAttributeFactory. See- * <a href="http://site.icu-project.org/charts/collation-icu4j-sun"- * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key- * generation timing and key length comparisons between ICU4J and- * java.text.Collator over several languages.- * </p>- * <p>- * CollationKeys generated by java.text.Collators are not compatible- * with those those generated by ICU Collators. Specifically, if you use- * CollationAttributeFactory to generate index terms, do not use- * ICUCollationAttributeFactory on the query side, or vice versa.- * </p>+ * <li>JVM vendor+ * <li>JVM version, including patch version+ * <li>The language (and country and variant, if specified) of the Locale used when constructing+ * the collator via {@link Collator#getInstance(java.util.Locale)}.+ * <li>The collation strength used - see {@link Collator#setStrength(int)}+ * </ol>+ *+ * <p>The <code>ICUCollationAttributeFactory</code> in the analysis-icu package uses ICU4J's+ * Collator, which makes its version available, thus allowing collation to be versioned+ * independently from the JVM. ICUCollationAttributeFactory is also significantly faster and+ * generates significantly shorter keys than CollationAttributeFactory. See <a+ * href="http://site.icu-project.org/charts/collation-icu4j-sun"+ * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key generation timing and key+ * length comparisons between ICU4J and java.text.Collator over several languages.+ *+ * <p>CollationKeys generated by java.text.Collators are not compatible with those those generated+ * by ICU Collators. Specifically, if you use CollationAttributeFactory to generate index terms, do+ * not use ICUCollationAttributeFactory on the query side, or vice versa.*/-public class CollationAttributeFactory extends AttributeFactory.StaticImplementationAttributeFactory<CollatedTermAttributeImpl> {+public class CollationAttributeFactory+ extends AttributeFactory.StaticImplementationAttributeFactory<CollatedTermAttributeImpl> {private final Collator collator;-+/**- * Create a CollationAttributeFactory, using- * {@link TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY} as the- * factory for all other attributes.+ * Create a CollationAttributeFactory, using {@link TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY}+ * as the factory for all other attributes.+ ** @param collator CollationKey generator*/public CollationAttributeFactory(Collator collator) {this(TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, collator);}-+/**- * Create a CollationAttributeFactory, using the supplied Attribute Factory- * as the factory for all other attributes.+ * Create a CollationAttributeFactory, using the supplied Attribute Factory as the factory for all+ * other attributes.+ ** @param delegate Attribute Factory* @param collator CollationKey generator*/@@ -90,7 +76,7 @@ public class CollationAttributeFactory extends AttributeFactory.StaticImplementasuper(delegate, CollatedTermAttributeImpl.class);this.collator = collator;}-+@Overridepublic CollatedTermAttributeImpl createInstance() {return new CollatedTermAttributeImpl(collator);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.javaindex 507abfd075a..f34285f888f 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationDocValuesField.java@@ -16,38 +16,35 @@*/package org.apache.lucene.collation;-import java.text.Collator;-import org.apache.lucene.document.Field;import org.apache.lucene.document.SortedDocValuesField;import org.apache.lucene.util.BytesRef;/*** Indexes collation keys as a single-valued {@link SortedDocValuesField}.- * <p>- * This is more efficient that {@link CollationKeyAnalyzer} if the field- * only has one value: no uninversion is necessary to sort on the field,- * locale-sensitive range queries can still work via {@code DocValuesRangeQuery},- * and the underlying data structures built at index-time are likely more efficient- * and use less memory than FieldCache.+ *+ * <p>This is more efficient that {@link CollationKeyAnalyzer} if the field only has one value: no+ * uninversion is necessary to sort on the field, locale-sensitive range queries can still work via+ * {@code DocValuesRangeQuery}, and the underlying data structures built at index-time are likely+ * more efficient and use less memory than FieldCache.*/public final class CollationDocValuesField extends Field {private final String name;private final Collator collator;private final BytesRef bytes = new BytesRef();-+/*** Create a new ICUCollationDocValuesField.- * <p>- * NOTE: you should not create a new one for each document, instead- * just make one and reuse it during your indexing process, setting- * the value via {@link #setStringValue(String)}.+ *+ * <p>NOTE: you should not create a new one for each document, instead just make one and reuse it+ * during your indexing process, setting the value via {@link #setStringValue(String)}.+ ** @param name field name* @param collator Collator for generating collation keys.*/// TODO: can we make this trap-free? maybe just synchronize on the collator- // instead?+ // instead?public CollationDocValuesField(String name, Collator collator) {super(name, SortedDocValuesField.TYPE);this.name = name;@@ -59,7 +56,7 @@ public final class CollationDocValuesField extends Field {public String name() {return name;}-+@Overridepublic void setStringValue(String value) {bytes.bytes = collator.getCollationKey(value).toByteArray();diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.javaindex d2099fb6323..fa8d6bae41b 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java@@ -16,67 +16,48 @@*/package org.apache.lucene.collation;--+import java.text.Collator;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.core.KeywordTokenizer;import org.apache.lucene.util.AttributeFactory;-import java.text.Collator;-/**- * <p>- * Configures {@link KeywordTokenizer} with {@link CollationAttributeFactory}.- * </p>- * <p>- * Converts the token into its {@link java.text.CollationKey}, and then- * encodes the CollationKey directly to allow- * it to be stored as an index term.- * </p>- * <p>- * <strong>WARNING:</strong> Make sure you use exactly the same Collator at- * index and query time -- CollationKeys are only comparable when produced by- * the same Collator. Since {@link java.text.RuleBasedCollator}s are not- * independently versioned, it is unsafe to search against stored- * CollationKeys unless the following are exactly the same (best practice is- * to store this information with the index and check that they remain the- * same at query time):- * </p>+ * Configures {@link KeywordTokenizer} with {@link CollationAttributeFactory}.+ *+ * <p>Converts the token into its {@link java.text.CollationKey}, and then encodes the CollationKey+ * directly to allow it to be stored as an index term.+ *+ * <p><strong>WARNING:</strong> Make sure you use exactly the same Collator at index and query time+ * -- CollationKeys are only comparable when produced by the same Collator. Since {@link+ * java.text.RuleBasedCollator}s are not independently versioned, it is unsafe to search against+ * stored CollationKeys unless the following are exactly the same (best practice is to store this+ * information with the index and check that they remain the same at query time):+ ** <ol>- * <li>JVM vendor</li>- * <li>JVM version, including patch version</li>- * <li>- * The language (and country and variant, if specified) of the Locale- * used when constructing the collator via- * {@link Collator#getInstance(java.util.Locale)}.- * </li>- * <li>- * The collation strength used - see {@link Collator#setStrength(int)}- * </li>- * </ol>- * <p>- * The <code>ICUCollationKeyAnalyzer</code> in the analysis-icu package- * uses ICU4J's Collator, which makes- * its version available, thus allowing collation to be versioned- * independently from the JVM. ICUCollationKeyAnalyzer is also significantly- * faster and generates significantly shorter keys than CollationKeyAnalyzer.- * See <a href="http://site.icu-project.org/charts/collation-icu4j-sun"- * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key- * generation timing and key length comparisons between ICU4J and- * java.text.Collator over several languages.- * </p>- * <p>- * CollationKeys generated by java.text.Collators are not compatible- * with those those generated by ICU Collators. Specifically, if you use- * CollationKeyAnalyzer to generate index terms, do not use- * ICUCollationKeyAnalyzer on the query side, or vice versa.- * </p>+ * <li>JVM vendor+ * <li>JVM version, including patch version+ * <li>The language (and country and variant, if specified) of the Locale used when constructing+ * the collator via {@link Collator#getInstance(java.util.Locale)}.+ * <li>The collation strength used - see {@link Collator#setStrength(int)}+ * </ol>+ *+ * <p>The <code>ICUCollationKeyAnalyzer</code> in the analysis-icu package uses ICU4J's Collator,+ * which makes its version available, thus allowing collation to be versioned independently from the+ * JVM. ICUCollationKeyAnalyzer is also significantly faster and generates significantly shorter+ * keys than CollationKeyAnalyzer. See <a+ * href="http://site.icu-project.org/charts/collation-icu4j-sun"+ * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key generation timing and key+ * length comparisons between ICU4J and java.text.Collator over several languages.+ *+ * <p>CollationKeys generated by java.text.Collators are not compatible with those those generated+ * by ICU Collators. Specifically, if you use CollationKeyAnalyzer to generate index terms, do not+ * use ICUCollationKeyAnalyzer on the query side, or vice versa.** @since 3.1*/public final class CollationKeyAnalyzer extends Analyzer {private final CollationAttributeFactory factory;-+/*** Create a new CollationKeyAnalyzer, using the specified collator.*@@ -93,7 +74,8 @@ public final class CollationKeyAnalyzer extends Analyzer {@Overrideprotected TokenStreamComponents createComponents(String fieldName) {- KeywordTokenizer tokenizer = new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);+ KeywordTokenizer tokenizer =+ new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);return new TokenStreamComponents(tokenizer, tokenizer);}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.javaindex c79b58cc088..f9c0ffc44df 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java@@ -16,35 +16,28 @@*//**- * Unicode collation support.- * <p>- * <code>Collation</code> converts each token into its binary <code>CollationKey</code>- * using the provided <code>Collator</code>, allowing it to be stored as an index term.- * </p>- *+ * Unicode collation support.+ *+ * <p><code>Collation</code> converts each token into its binary <code>CollationKey</code> using the+ * provided <code>Collator</code>, allowing it to be stored as an index term.+ ** <h2>Use Cases</h2>- *+ ** <ul>- * <li>- * Efficient sorting of terms in languages that use non-Unicode character- * orderings. (Lucene Sort using a Locale can be very slow.)- * </li>- * <li>- * Efficient range queries over fields that contain terms in languages that- * use non-Unicode character orderings. (Range queries using a Locale can be- * very slow.)- * </li>- * <li>- * Effective Locale-specific normalization (case differences, diacritics, etc.).- * ({@link org.apache.lucene.analysis.LowerCaseFilter} and- * {@link org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} provide these services- * in a generic way that doesn't take into account locale-specific needs.)- * </li>+ * <li>Efficient sorting of terms in languages that use non-Unicode character orderings. (Lucene+ * Sort using a Locale can be very slow.)+ * <li>Efficient range queries over fields that contain terms in languages that use non-Unicode+ * character orderings. (Range queries using a Locale can be very slow.)+ * <li>Effective Locale-specific normalization (case differences, diacritics, etc.). ({@link+ * org.apache.lucene.analysis.LowerCaseFilter} and {@link+ * org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} provide these services in a+ * generic way that doesn't take into account locale-specific needs.)* </ul>- *+ ** <h2>Example Usages</h2>- *+ ** <h3>Farsi Range Queries</h3>+ ** <pre class="prettyprint">* // "fa" Locale is not supported by Sun JDK 1.4 or 1.5* Collator collator = Collator.getInstance(new Locale("ar"));@@ -58,10 +51,10 @@* writer.close();* IndexReader ir = DirectoryReader.open(dir);* IndexSearcher is = new IndexSearcher(ir);- *+ ** QueryParser aqp = new QueryParser("content", analyzer);* aqp.setAnalyzeRangeTerms(true);- *+ ** // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi* // orders the U+0698 character before the U+0633 character, so the single* // indexed Term above should NOT be returned by a ConstantScoreRangeQuery@@ -71,10 +64,11 @@* = is.search(aqp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs;* assertEquals("The index Term should not be included.", 0, result.length);* </pre>- *+ ** <h3>Danish Sorting</h3>+ ** <pre class="prettyprint">- * Analyzer analyzer+ * Analyzer analyzer* = new CollationKeyAnalyzer(Collator.getInstance(new Locale("da", "dk")));* Path dirPath = Files.createTempDirectory("tempIndex");* Directory dir = FSDirectory.open(dirPath);@@ -100,8 +94,9 @@* assertEquals(sortedTracerOrder[i], doc.getValues("tracer")[0]);* }* </pre>- *+ ** <h3>Turkish Case Normalization</h3>+ ** <pre class="prettyprint">* Collator collator = Collator.getInstance(new Locale("tr", "TR"));* collator.setStrength(Collator.PRIMARY);@@ -120,47 +115,36 @@* ScoreDoc[] result = is.search(query, null, 1000).scoreDocs;* assertEquals("The index Term should be included.", 1, result.length);* </pre>- *+ ** <h2>Caveats and Comparisons</h2>- * <p>- * <strong>WARNING:</strong> Make sure you use exactly the same- * <code>Collator</code> at index and query time -- <code>CollationKey</code>s- * are only comparable when produced by- * the same <code>Collator</code>. Since {@link java.text.RuleBasedCollator}s- * are not independently versioned, it is unsafe to search against stored- * <code>CollationKey</code>s unless the following are exactly the same (best- * practice is to store this information with the index and check that they- * remain the same at query time):- * </p>+ *+ * <p><strong>WARNING:</strong> Make sure you use exactly the same <code>Collator</code> at index+ * and query time -- <code>CollationKey</code>s are only comparable when produced by the same <code>+ * Collator</code>. Since {@link java.text.RuleBasedCollator}s are not independently versioned, it+ * is unsafe to search against stored <code>CollationKey</code>s unless the following are exactly+ * the same (best practice is to store this information with the index and check that they remain+ * the same at query time):+ ** <ol>- * <li>JVM vendor</li>- * <li>JVM version, including patch version</li>- * <li>- * The language (and country and variant, if specified) of the Locale- * used when constructing the collator via- * {@link java.text.Collator#getInstance(java.util.Locale)}.- * </li>- * <li>- * The collation strength used - see {@link java.text.Collator#setStrength(int)}- * </li>- * </ol>- * <p>- * <code>ICUCollationKeyAnalyzer</code>, available in the <a href="{@docRoot}/../icu/overview-summary.html">icu analysis module</a>,- * uses ICU4J's <code>Collator</code>, which- * makes its version available, thus allowing collation to be versioned- * independently from the JVM. <code>ICUCollationKeyAnalyzer</code> is also- * significantly faster and generates significantly shorter keys than- * <code>CollationKeyAnalyzer</code>. See- * <a href="http://site.icu-project.org/charts/collation-icu4j-sun"- * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key- * generation timing and key length comparisons between ICU4J and- * <code>java.text.Collator</code> over several languages.- * </p>- * <p>- * <code>CollationKey</code>s generated by <code>java.text.Collator</code>s are- * not compatible with those those generated by ICU Collators. Specifically, if- * you use <code>CollationKeyAnalyzer</code> to generate index terms, do not use- * <code>ICUCollationKeyAnalyzer</code> on the query side, or vice versa.- * </p>+ * <li>JVM vendor+ * <li>JVM version, including patch version+ * <li>The language (and country and variant, if specified) of the Locale used when constructing+ * the collator via {@link java.text.Collator#getInstance(java.util.Locale)}.+ * <li>The collation strength used - see {@link java.text.Collator#setStrength(int)}+ * </ol>+ *+ * <p><code>ICUCollationKeyAnalyzer</code>, available in the <a+ * href="{@docRoot}/../icu/overview-summary.html">icu analysis module</a>, uses ICU4J's <code>+ * Collator</code>, which makes its version available, thus allowing collation to be versioned+ * independently from the JVM. <code>ICUCollationKeyAnalyzer</code> is also significantly faster and+ * generates significantly shorter keys than <code>CollationKeyAnalyzer</code>. See <a+ * href="http://site.icu-project.org/charts/collation-icu4j-sun"+ * >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key generation timing and key+ * length comparisons between ICU4J and <code>java.text.Collator</code> over several languages.+ *+ * <p><code>CollationKey</code>s generated by <code>java.text.Collator</code>s are not compatible+ * with those those generated by ICU Collators. Specifically, if you use <code>CollationKeyAnalyzer+ * </code> to generate index terms, do not use <code>ICUCollationKeyAnalyzer</code> on the query+ * side, or vice versa.*/package org.apache.lucene.collation;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.javaindex d997505c6e9..aa3b082bf78 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java@@ -16,21 +16,20 @@*/package org.apache.lucene.collation.tokenattributes;-import java.text.Collator;-import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;import org.apache.lucene.util.BytesRef;/**- * Extension of {@link CharTermAttributeImpl} that encodes the term- * text as a binary Unicode collation key instead of as UTF-8 bytes.+ * Extension of {@link CharTermAttributeImpl} that encodes the term text as a binary Unicode+ * collation key instead of as UTF-8 bytes.*/public class CollatedTermAttributeImpl extends CharTermAttributeImpl {private final Collator collator;/*** Create a new CollatedTermAttributeImpl+ ** @param collator Collation key generator*/public CollatedTermAttributeImpl(Collator collator) {@@ -38,7 +37,7 @@ public class CollatedTermAttributeImpl extends CharTermAttributeImpl {// or to reduce contention in case they dothis.collator = (Collator) collator.clone();}-+@Overridepublic BytesRef getBytesRef() {final BytesRef ref = this.builder.get();@@ -47,5 +46,4 @@ public class CollatedTermAttributeImpl extends CharTermAttributeImpl {ref.length = ref.bytes.length;return ref;}-}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.javaindex 2ace0ee670f..a4591d0a8a9 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java@@ -16,99 +16,85 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;-import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.CharArraySet;-/**- * Test the Arabic Analyzer- *- */+/** Test the Arabic Analyzer */public class TestArabicAnalyzer extends BaseTokenStreamTestCase {-- /** This test fails with NPE when the- * stopwords file is missing in classpath */++ /** This test fails with NPE when the stopwords file is missing in classpath */public void testResourcesAvailable() {new ArabicAnalyzer().close();}-+/*** Some simple tests showing some features of the analyzer, how some regular forms will conflate*/public void testBasicFeatures() throws Exception {ArabicAnalyzer a = new ArabicAnalyzer();- assertAnalyzesTo(a, "كبير", new String[] { "كبير" });- assertAnalyzesTo(a, "كبيرة", new String[] { "كبير" }); // feminine marker-- assertAnalyzesTo(a, "مشروب", new String[] { "مشروب" });- assertAnalyzesTo(a, "مشروبات", new String[] { "مشروب" }); // plural -at-- assertAnalyzesTo(a, "أمريكيين", new String[] { "امريك" }); // plural -in- assertAnalyzesTo(a, "امريكي", new String[] { "امريك" }); // singular with bare alif-- assertAnalyzesTo(a, "كتاب", new String[] { "كتاب" });- assertAnalyzesTo(a, "الكتاب", new String[] { "كتاب" }); // definite article-- assertAnalyzesTo(a, "ما ملكت أيمانكم", new String[] { "ملكت", "ايمانكم"});- assertAnalyzesTo(a, "الذين ملكت أيمانكم", new String[] { "ملكت", "ايمانكم" }); // stopwords+ assertAnalyzesTo(a, "كبير", new String[] {"كبير"});+ assertAnalyzesTo(a, "كبيرة", new String[] {"كبير"}); // feminine marker++ assertAnalyzesTo(a, "مشروب", new String[] {"مشروب"});+ assertAnalyzesTo(a, "مشروبات", new String[] {"مشروب"}); // plural -at++ assertAnalyzesTo(a, "أمريكيين", new String[] {"امريك"}); // plural -in+ assertAnalyzesTo(a, "امريكي", new String[] {"امريك"}); // singular with bare alif++ assertAnalyzesTo(a, "كتاب", new String[] {"كتاب"});+ assertAnalyzesTo(a, "الكتاب", new String[] {"كتاب"}); // definite article++ assertAnalyzesTo(a, "ما ملكت أيمانكم", new String[] {"ملكت", "ايمانكم"});+ assertAnalyzesTo(a, "الذين ملكت أيمانكم", new String[] {"ملكت", "ايمانكم"}); // stopwordsa.close();}-- /**- * Simple tests to show things are getting reset correctly, etc.- */++ /** Simple tests to show things are getting reset correctly, etc. */public void testReusableTokenStream() throws Exception {ArabicAnalyzer a = new ArabicAnalyzer();- assertAnalyzesTo(a, "كبير", new String[] { "كبير" });- assertAnalyzesTo(a, "كبيرة", new String[] { "كبير" }); // feminine marker+ assertAnalyzesTo(a, "كبير", new String[] {"كبير"});+ assertAnalyzesTo(a, "كبيرة", new String[] {"كبير"}); // feminine markera.close();}- /**- * Non-arabic text gets treated in a similar way as SimpleAnalyzer.- */+ /** Non-arabic text gets treated in a similar way as SimpleAnalyzer. */public void testEnglishInput() throws Exception {ArabicAnalyzer a = new ArabicAnalyzer();- assertAnalyzesTo(a, "English text.", new String[] {- "english", "text" });+ assertAnalyzesTo(a, "English text.", new String[] {"english", "text"});a.close();}-- /**- * Test that custom stopwords work, and are not case-sensitive.- */++ /** Test that custom stopwords work, and are not case-sensitive. */public void testCustomStopwords() throws Exception {CharArraySet set = new CharArraySet(asSet("the", "and", "a"), false);ArabicAnalyzer a = new ArabicAnalyzer(set);- assertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick",- "brown", "fox" });+ assertAnalyzesTo(a, "The quick brown fox.", new String[] {"quick", "brown", "fox"});a.close();}-+public void testWithStemExclusionSet() throws IOException {CharArraySet set = new CharArraySet(asSet("ساهدهات"), false);ArabicAnalyzer a = new ArabicAnalyzer(CharArraySet.EMPTY_SET, set);- assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهدهات" });- assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهدهات" });+ assertAnalyzesTo(+ a, "كبيرة the quick ساهدهات", new String[] {"كبير", "the", "quick", "ساهدهات"});+ assertAnalyzesTo(+ a, "كبيرة the quick ساهدهات", new String[] {"كبير", "the", "quick", "ساهدهات"});a.close();-+a = new ArabicAnalyzer(CharArraySet.EMPTY_SET, CharArraySet.EMPTY_SET);- assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهد" });- assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] { "كبير","the", "quick", "ساهد" });+ assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] {"كبير", "the", "quick", "ساهد"});+ assertAnalyzesTo(a, "كبيرة the quick ساهدهات", new String[] {"كبير", "the", "quick", "ساهد"});a.close();}-- /**- * test we fold digits to latin-1- */++ /** test we fold digits to latin-1 */public void testDigits() throws Exception {ArabicAnalyzer a = new ArabicAnalyzer();checkOneTerm(a, "١٢٣٤", "1234");a.close();}-+/** blast some random strings through the analyzer */public void testRandomStrings() throws Exception {ArabicAnalyzer a = new ArabicAnalyzer();diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.javaindex f3f833d8337..246210e76f6 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java@@ -16,32 +16,24 @@*/package org.apache.lucene.analysis.ar;-import java.io.Reader;import java.io.StringReader;-+import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;-import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;-/**- * Simple tests to ensure the Arabic filter Factories are working.- */+/** Simple tests to ensure the Arabic filter Factories are working. */public class TestArabicFilters extends BaseTokenStreamFactoryTestCase {-- /**- * Test ArabicNormalizationFilterFactory- */++ /** Test ArabicNormalizationFilterFactory */public void testNormalizer() throws Exception {Reader reader = new StringReader("الذين مَلكت أيمانكم");Tokenizer tokenizer = whitespaceMockTokenizer(reader);TokenStream stream = tokenFilterFactory("ArabicNormalization").create(tokenizer);assertTokenStreamContents(stream, new String[] {"الذين", "ملكت", "ايمانكم"});}-- /**- * Test ArabicStemFilterFactory- */++ /** Test ArabicStemFilterFactory */public void testStemmer() throws Exception {Reader reader = new StringReader("الذين مَلكت أيمانكم");Tokenizer tokenizer = whitespaceMockTokenizer(reader);@@ -49,31 +41,38 @@ public class TestArabicFilters extends BaseTokenStreamFactoryTestCase {stream = tokenFilterFactory("ArabicStem").create(stream);assertTokenStreamContents(stream, new String[] {"ذين", "ملكت", "ايمانكم"});}-- /**- * Test PersianCharFilterFactory- */++ /** Test PersianCharFilterFactory */public void testPersianCharFilter() throws Exception {Reader reader = charFilterFactory("Persian").create(new StringReader("میخورد"));Tokenizer tokenizer = whitespaceMockTokenizer(reader);- assertTokenStreamContents(tokenizer, new String[] { "می", "خورد" });+ assertTokenStreamContents(tokenizer, new String[] {"می", "خورد"});}-+/** Test that bogus arguments result in exception */public void testBogusArguments() throws Exception {- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {- tokenFilterFactory("ArabicNormalization", "bogusArg", "bogusValue");- });+ IllegalArgumentException expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ tokenFilterFactory("ArabicNormalization", "bogusArg", "bogusValue");+ });assertTrue(expected.getMessage().contains("Unknown parameters"));-- expected = expectThrows(IllegalArgumentException.class, () -> {- tokenFilterFactory("Arabicstem", "bogusArg", "bogusValue");- });++ expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ tokenFilterFactory("Arabicstem", "bogusArg", "bogusValue");+ });assertTrue(expected.getMessage().contains("Unknown parameters"));-- expected = expectThrows(IllegalArgumentException.class, () -> {- charFilterFactory("Persian", "bogusArg", "bogusValue");- });++ expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ charFilterFactory("Persian", "bogusArg", "bogusValue");+ });assertTrue(expected.getMessage().contains("Unknown parameters"));}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.javaindex a0409019db8..d833b3047c3 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java@@ -16,33 +16,29 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;import java.io.StringReader;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.MockTokenizer;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.KeywordTokenizer;-/**- * Test the Arabic Normalization Filter- */+/** Test the Arabic Normalization Filter */public class TestArabicNormalizationFilter extends BaseTokenStreamTestCase {public void testAlifMadda() throws IOException {check("آجن", "اجن");}-+public void testAlifHamzaAbove() throws IOException {check("أحمد", "احمد");}-+public void testAlifHamzaBelow() throws IOException {check("إعاذ", "اعاذ");}-+public void testAlifMaksura() throws IOException {check("بنى", "بني");}@@ -50,60 +46,60 @@ public class TestArabicNormalizationFilter extends BaseTokenStreamTestCase {public void testTehMarbuta() throws IOException {check("فاطمة", "فاطمه");}-+public void testTatweel() throws IOException {check("روبرـــــت", "روبرت");}-+public void testFatha() throws IOException {check("مَبنا", "مبنا");}-+public void testKasra() throws IOException {check("علِي", "علي");}-+public void testDamma() throws IOException {check("بُوات", "بوات");}-+public void testFathatan() throws IOException {check("ولداً", "ولدا");}-+public void testKasratan() throws IOException {check("ولدٍ", "ولد");}-+public void testDammatan() throws IOException {check("ولدٌ", "ولد");- }-+ }+public void testSukun() throws IOException {check("نلْسون", "نلسون");}-+public void testShaddah() throws IOException {check("هتميّ", "هتمي");- }-+ }+private void check(final String input, final String expected) throws IOException {MockTokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);tokenStream.setReader(new StringReader(input));ArabicNormalizationFilter filter = new ArabicNormalizationFilter(tokenStream);- assertTokenStreamContents(filter, new String[]{expected});+ assertTokenStreamContents(filter, new String[] {expected});}-+public void testEmptyTerm() throws IOException {- Analyzer a = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new KeywordTokenizer();- return new TokenStreamComponents(tokenizer, new ArabicNormalizationFilter(tokenizer));- }- };+ Analyzer a =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new KeywordTokenizer();+ return new TokenStreamComponents(tokenizer, new ArabicNormalizationFilter(tokenizer));+ }+ };checkOneTerm(a, "", "");a.close();}-}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.javaindex 872e7f59c8f..403a1af20ea 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java@@ -16,9 +16,7 @@*/package org.apache.lucene.analysis.ar;-import java.io.IOException;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.CharArraySet;@@ -27,88 +25,85 @@ import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.KeywordTokenizer;import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;-/**- * Test the Arabic Normalization Filter- *- */+/** Test the Arabic Normalization Filter */public class TestArabicStemFilter extends BaseTokenStreamTestCase {-+public void testAlPrefix() throws IOException {check("الحسن", "حسن");- }+ }public void testWalPrefix() throws IOException {check("والحسن", "حسن");- }-+ }+public void testBalPrefix() throws IOException {check("بالحسن", "حسن");- }-+ }+public void testKalPrefix() throws IOException {check("كالحسن", "حسن");- }-+ }+public void testFalPrefix() throws IOException {check("فالحسن", "حسن");- }+ }public void testLlPrefix() throws IOException {- check("للاخر", "اخر");+ check("للاخر", "اخر");}-+public void testWaPrefix() throws IOException {check("وحسن", "حسن");- }-+ }+public void testAhSuffix() throws IOException {check("زوجها", "زوج");- }-+ }+public void testAnSuffix() throws IOException {check("ساهدان", "ساهد");- }-+ }+public void testAtSuffix() throws IOException {check("ساهدات", "ساهد");- }-+ }+public void testWnSuffix() throws IOException {check("ساهدون", "ساهد");- }-+ }+public void testYnSuffix() throws IOException {check("ساهدين", "ساهد");- }-+ }+public void testYhSuffix() throws IOException {check("ساهديه", "ساهد");- }+ }public void testYpSuffix() throws IOException {check("ساهدية", "ساهد");- }-+ }+public void testHSuffix() throws IOException {check("ساهده", "ساهد");- }-+ }+public void testPSuffix() throws IOException {check("ساهدة", "ساهد");}-+public void testYSuffix() throws IOException {check("ساهدي", "ساهد");}-+public void testComboPrefSuf() throws IOException {check("وساهدون", "ساهد");}-+public void testComboSuf() throws IOException {check("ساهدهات", "ساهد");}-+public void testShouldntStem() throws IOException {check("الو", "الو");}@@ -116,30 +111,31 @@ public class TestArabicStemFilter extends BaseTokenStreamTestCase {public void testNonArabic() throws IOException {check("English", "English");}-+public void testWithKeywordAttribute() throws IOException {CharArraySet set = new CharArraySet(1, true);set.add("ساهدهات");- MockTokenizer tokenStream = whitespaceMockTokenizer("ساهدهات");+ MockTokenizer tokenStream = whitespaceMockTokenizer("ساهدهات");ArabicStemFilter filter = new ArabicStemFilter(new SetKeywordMarkerFilter(tokenStream, set));- assertTokenStreamContents(filter, new String[]{"ساهدهات"});+ assertTokenStreamContents(filter, new String[] {"ساهدهات"});}private void check(final String input, final String expected) throws IOException {- MockTokenizer tokenStream = whitespaceMockTokenizer(input);+ MockTokenizer tokenStream = whitespaceMockTokenizer(input);ArabicStemFilter filter = new ArabicStemFilter(tokenStream);- assertTokenStreamContents(filter, new String[]{expected});+ assertTokenStreamContents(filter, new String[] {expected});}-+public void testEmptyTerm() throws IOException {- Analyzer a = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new KeywordTokenizer();- return new TokenStreamComponents(tokenizer, new ArabicStemFilter(tokenizer));- }- };+ Analyzer a =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new KeywordTokenizer();+ return new TokenStreamComponents(tokenizer, new ArabicStemFilter(tokenizer));+ }+ };checkOneTerm(a, "", "");a.close();}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.javaindex 50f3ab939f6..c62cee437c1 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java@@ -16,10 +16,8 @@*/package org.apache.lucene.collation;-import java.text.Collator;import java.util.Locale;-import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.StringField;@@ -37,42 +35,41 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.TestUtil;-/**- * trivial test of CollationDocValuesField- */+/** trivial test of CollationDocValuesField */public class TestCollationDocValuesField extends LuceneTestCase {-+public void testBasic() throws Exception {Directory dir = newDirectory();RandomIndexWriter iw = new RandomIndexWriter(random(), dir);Document doc = new Document();Field field = newField("field", "", StringField.TYPE_STORED);- CollationDocValuesField collationField = new CollationDocValuesField("collated", Collator.getInstance(Locale.ENGLISH));+ CollationDocValuesField collationField =+ new CollationDocValuesField("collated", Collator.getInstance(Locale.ENGLISH));doc.add(field);doc.add(collationField);field.setStringValue("ABC");collationField.setStringValue("ABC");iw.addDocument(doc);-+field.setStringValue("abc");collationField.setStringValue("abc");iw.addDocument(doc);-+IndexReader ir = iw.getReader();iw.close();-+IndexSearcher is = newSearcher(ir);-+SortField sortField = new SortField("collated", SortField.Type.STRING);-+TopDocs td = is.search(new MatchAllDocsQuery(), 5, new Sort(sortField));assertEquals("abc", ir.document(td.scoreDocs[0].doc).get("field"));assertEquals("ABC", ir.document(td.scoreDocs[1].doc).get("field"));ir.close();dir.close();}-+public void testRanges() throws Exception {Directory dir = newDirectory();RandomIndexWriter iw = new RandomIndexWriter(random(), dir);@@ -85,7 +82,7 @@ public class TestCollationDocValuesField extends LuceneTestCase {CollationDocValuesField collationField = new CollationDocValuesField("collated", collator);doc.add(field);doc.add(collationField);-+int numDocs = atLeast(100);for (int i = 0; i < numDocs; i++) {String value = TestUtil.randomSimpleString(random());@@ -93,13 +90,13 @@ public class TestCollationDocValuesField extends LuceneTestCase {collationField.setStringValue(value);iw.addDocument(doc);}-+IndexReader ir = iw.getReader();iw.close();IndexSearcher is = newSearcher(ir);-+int numChecks = atLeast(20);-+try {for (int i = 0; i < numChecks; i++) {String start = TestUtil.randomSimpleString(random());@@ -113,13 +110,21 @@ public class TestCollationDocValuesField extends LuceneTestCase {dir.close();}}-- private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception {++ private void doTestRanges(+ IndexSearcher is,+ String startPoint,+ String endPoint,+ BytesRef startBR,+ BytesRef endBR,+ Collator collator)+ throws Exception {SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");- for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {+ for (int docID = 0; docID < is.getIndexReader().maxDoc(); docID++) {Document doc = is.doc(docID);String s = doc.getField("field").stringValue();- boolean collatorAccepts = collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;+ boolean collatorAccepts =+ collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;assertEquals(docID, dvs.nextDoc());BytesRef br = dvs.binaryValue();boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;diff --git a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.javaindex b1a7fb288a5..efc3c2df256 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java@@ -16,56 +16,55 @@*/package org.apache.lucene.collation;-+import java.text.Collator;+import java.util.Locale;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CollationTestBase;import org.apache.lucene.util.BytesRef;-import java.text.Collator;-import java.util.Locale;--public class TestCollationKeyAnalyzer extends CollationTestBase {+public class TestCollationKeyAnalyzer extends CollationTestBase {// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in// RuleBasedCollator. However, the Arabic Locale seems to order the Farsi// characters properly.private Collator collator = Collator.getInstance(new Locale("ar"));private Analyzer analyzer;-+@Overridepublic void setUp() throws Exception {super.setUp();analyzer = new CollationKeyAnalyzer(collator);}-+@Overridepublic void tearDown() throws Exception {analyzer.close();super.tearDown();}- private BytesRef firstRangeBeginning = new BytesRef(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());- private BytesRef firstRangeEnd = new BytesRef(collator.getCollationKey(firstRangeEndOriginal).toByteArray());- private BytesRef secondRangeBeginning = new BytesRef(collator.getCollationKey(secondRangeBeginningOriginal).toByteArray());- private BytesRef secondRangeEnd = new BytesRef(collator.getCollationKey(secondRangeEndOriginal).toByteArray());+ private BytesRef firstRangeBeginning =+ new BytesRef(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());+ private BytesRef firstRangeEnd =+ new BytesRef(collator.getCollationKey(firstRangeEndOriginal).toByteArray());+ private BytesRef secondRangeBeginning =+ new BytesRef(collator.getCollationKey(secondRangeBeginningOriginal).toByteArray());+ private BytesRef secondRangeEnd =+ new BytesRef(collator.getCollationKey(secondRangeEndOriginal).toByteArray());public void testFarsiRangeFilterCollating() throws Exception {- testFarsiRangeFilterCollating- (analyzer, firstRangeBeginning, firstRangeEnd,- secondRangeBeginning, secondRangeEnd);+ testFarsiRangeFilterCollating(+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);}-+public void testFarsiRangeQueryCollating() throws Exception {- testFarsiRangeQueryCollating- (analyzer, firstRangeBeginning, firstRangeEnd,- secondRangeBeginning, secondRangeEnd);+ testFarsiRangeQueryCollating(+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);}public void testFarsiTermRangeQuery() throws Exception {- testFarsiTermRangeQuery- (analyzer, firstRangeBeginning, firstRangeEnd,- secondRangeBeginning, secondRangeEnd);+ testFarsiTermRangeQuery(+ analyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd);}-+public void testThreadSafe() throws Exception {int iters = 20 * RANDOM_MULTIPLIER;for (int i = 0; i < iters; i++) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.javaindex 3366611dd4a..ef3d5c2ffb3 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java@@ -16,10 +16,8 @@*/package org.apache.lucene.analysis.standard;-import java.io.IOException;import java.io.Reader;-import org.apache.lucene.analysis.CharArraySet;import org.apache.lucene.analysis.LowerCaseFilter;import org.apache.lucene.analysis.StopFilter;@@ -28,51 +26,57 @@ import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.WordlistLoader;/**- * Filters {@link StandardTokenizer} with {@link LowerCaseFilter} and- * {@link StopFilter}, using a configurable list of stop words.+ * Filters {@link StandardTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}, using a+ * configurable list of stop words.** @since 3.1*/public final class StandardAnalyzer extends StopwordAnalyzerBase {-+/** Default maximum allowed token length */public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;- /** Builds an analyzer with the given stop words.- * @param stopWords stop words */+ /**+ * Builds an analyzer with the given stop words.+ *+ * @param stopWords stop words+ */public StandardAnalyzer(CharArraySet stopWords) {super(stopWords);}- /** Builds an analyzer with no stop words.- */+ /** Builds an analyzer with no stop words. */public StandardAnalyzer() {this(CharArraySet.EMPTY_SET);}- /** Builds an analyzer with the stop words from the given reader.+ /**+ * Builds an analyzer with the stop words from the given reader.+ ** @see WordlistLoader#getWordSet(Reader)- * @param stopwords Reader to read stop words from */+ * @param stopwords Reader to read stop words from+ */public StandardAnalyzer(Reader stopwords) throws IOException {this(loadStopwordSet(stopwords));}/**- * Set the max allowed token length. Tokens larger than this will be chopped- * up at this token length and emitted as multiple tokens. If you need to- * skip such large tokens, you could increase this max length, and then- * use {@code LengthFilter} to remove long tokens. The default is- * {@link StandardAnalyzer#DEFAULT_MAX_TOKEN_LENGTH}.+ * Set the max allowed token length. Tokens larger than this will be chopped up at this token+ * length and emitted as multiple tokens. If you need to skip such large tokens, you could+ * increase this max length, and then use {@code LengthFilter} to remove long tokens. The default+ * is {@link StandardAnalyzer#DEFAULT_MAX_TOKEN_LENGTH}.*/public void setMaxTokenLength(int length) {maxTokenLength = length;}-- /** Returns the current maximum token length- *- * @see #setMaxTokenLength */++ /**+ * Returns the current maximum token length+ *+ * @see #setMaxTokenLength+ */public int getMaxTokenLength() {return maxTokenLength;}@@ -83,10 +87,12 @@ public final class StandardAnalyzer extends StopwordAnalyzerBase {src.setMaxTokenLength(maxTokenLength);TokenStream tok = new LowerCaseFilter(src);tok = new StopFilter(tok, stopwords);- return new TokenStreamComponents(r -> {- src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);- src.setReader(r);- }, tok);+ return new TokenStreamComponents(+ r -> {+ src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);+ src.setReader(r);+ },+ tok);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.javaindex 50d1f9fb5d0..9cd7fbcc21a 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java@@ -18,7 +18,6 @@package org.apache.lucene.analysis.standard;import java.io.IOException;-import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;@@ -26,16 +25,16 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.analysis.tokenattributes.TypeAttribute;import org.apache.lucene.util.AttributeFactory;-/** A grammar-based tokenizer constructed with JFlex.- * <p>- * This class implements the Word Break rules from the- * Unicode Text Segmentation algorithm, as specified in- * <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.- * <p>Many applications have specific tokenizer needs. If this tokenizer does- * not suit your application, please consider copying this source code- * directory to your project and maintaining your own grammar-based tokenizer.+/**+ * A grammar-based tokenizer constructed with JFlex.+ *+ * <p>This class implements the Word Break rules from the Unicode Text Segmentation algorithm, as+ * specified in <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.+ *+ * <p>Many applications have specific tokenizer needs. If this tokenizer does not suit your+ * application, please consider copying this source code directory to your project and maintaining+ * your own grammar-based tokenizer.*/-public final class StandardTokenizer extends Tokenizer {/** A private instance of the JFlex-constructed scanner */private StandardTokenizerImpl scanner;@@ -56,36 +55,36 @@ public final class StandardTokenizer extends Tokenizer {public static final int HANGUL = 6;/** Emoji token type. */public static final int EMOJI = 7;-+/** String token types that correspond to token type int constants */- public static final String [] TOKEN_TYPES = new String [] {- "<ALPHANUM>",- "<NUM>",- "<SOUTHEAST_ASIAN>",- "<IDEOGRAPHIC>",- "<HIRAGANA>",- "<KATAKANA>",- "<HANGUL>",- "<EMOJI>"- };-+ public static final String[] TOKEN_TYPES =+ new String[] {+ "<ALPHANUM>",+ "<NUM>",+ "<SOUTHEAST_ASIAN>",+ "<IDEOGRAPHIC>",+ "<HIRAGANA>",+ "<KATAKANA>",+ "<HANGUL>",+ "<EMOJI>"+ };+/** Absolute maximum sized token */public static final int MAX_TOKEN_LENGTH_LIMIT = 1024 * 1024;-+private int skippedPositions;private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;/**- * Set the max allowed token length. Tokens larger than this will be chopped- * up at this token length and emitted as multiple tokens. If you need to- * skip such large tokens, you could increase this max length, and then- * use {@code LengthFilter} to remove long tokens. The default is- * {@link StandardAnalyzer#DEFAULT_MAX_TOKEN_LENGTH}.- *- * @throws IllegalArgumentException if the given length is outside of the- * range [1, {@value #MAX_TOKEN_LENGTH_LIMIT}].- */+ * Set the max allowed token length. Tokens larger than this will be chopped up at this token+ * length and emitted as multiple tokens. If you need to skip such large tokens, you could+ * increase this max length, and then use {@code LengthFilter} to remove long tokens. The default+ * is {@link StandardAnalyzer#DEFAULT_MAX_TOKEN_LENGTH}.+ *+ * @throws IllegalArgumentException if the given length is outside of the range [1, {@value+ * #MAX_TOKEN_LENGTH_LIMIT}].+ */public void setMaxTokenLength(int length) {if (length < 1) {throw new IllegalArgumentException("maxTokenLength must be greater than zero");@@ -98,25 +97,27 @@ public final class StandardTokenizer extends Tokenizer {}}- /** Returns the current maximum token length- *- * @see #setMaxTokenLength */+ /**+ * Returns the current maximum token length+ *+ * @see #setMaxTokenLength+ */public int getMaxTokenLength() {return maxTokenLength;}/**- * Creates a new instance of the {@link org.apache.lucene.analysis.standard.StandardTokenizer}. Attaches- * the <code>input</code> to the newly created JFlex scanner.-- * See http://issues.apache.org/jira/browse/LUCENE-1068+ * Creates a new instance of the {@link org.apache.lucene.analysis.standard.StandardTokenizer}.+ * Attaches the <code>input</code> to the newly created JFlex scanner.+ *+ * <p>See http://issues.apache.org/jira/browse/LUCENE-1068*/public StandardTokenizer() {init();}/**- * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}+ * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}*/public StandardTokenizer(AttributeFactory factory) {super(factory);@@ -131,7 +132,8 @@ public final class StandardTokenizer extends Tokenizer {// term offset, positionIncrement and typeprivate final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute posIncrAtt =+ addAttribute(PositionIncrementAttribute.class);private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);/*@@ -144,7 +146,7 @@ public final class StandardTokenizer extends Tokenizer {clearAttributes();skippedPositions = 0;- while(true) {+ while (true) {int tokenType = scanner.getNextToken();if (tokenType == StandardTokenizerImpl.YYEOF) {@@ -152,10 +154,10 @@ public final class StandardTokenizer extends Tokenizer {}if (scanner.yylength() <= maxTokenLength) {- posIncrAtt.setPositionIncrement(skippedPositions+1);+ posIncrAtt.setPositionIncrement(skippedPositions + 1);scanner.getText(termAtt);final int start = scanner.yychar();- offsetAtt.setOffset(correctOffset(start), correctOffset(start+termAtt.length()));+ offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));typeAtt.setType(StandardTokenizer.TOKEN_TYPES[tokenType]);return true;} else@@ -164,7 +166,7 @@ public final class StandardTokenizer extends Tokenizer {skippedPositions++;}}-+@Overridepublic final void end() throws IOException {super.end();@@ -172,7 +174,7 @@ public final class StandardTokenizer extends Tokenizer {int finalOffset = correctOffset(scanner.yychar() + scanner.yylength());offsetAtt.setOffset(finalOffset, finalOffset);// adjust any skipped tokens- posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement()+skippedPositions);+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.javaindex cfcf74c8e2e..ea8273a2758 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java@@ -16,20 +16,19 @@*/package org.apache.lucene.analysis.standard;-+import java.util.Map;import org.apache.lucene.analysis.TokenizerFactory;import org.apache.lucene.util.AttributeFactory;-import java.util.Map;-/**- * Factory for {@link StandardTokenizer}.+ * Factory for {@link StandardTokenizer}.+ ** <pre class="prettyprint">* <fieldType name="text_stndrd" class="solr.TextField" positionIncrementGap="100">* <analyzer>* <tokenizer class="solr.StandardTokenizerFactory" maxTokenLength="255"/>* </analyzer>- * </fieldType></pre>+ * </fieldType></pre>** @since 3.1* @lucene.spi {@value #NAME}@@ -40,9 +39,9 @@ public class StandardTokenizerFactory extends TokenizerFactory {public static final String NAME = "standard";private final int maxTokenLength;-+/** Creates a new StandardTokenizerFactory */- public StandardTokenizerFactory(Map<String,String> args) {+ public StandardTokenizerFactory(Map<String, String> args) {super(args);maxTokenLength = getInt(args, "maxTokenLength", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);if (!args.isEmpty()) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/standard/package-info.java b/lucene/core/src/java/org/apache/lucene/analysis/standard/package-info.javaindex ff313a3a939..aa133495f49 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/standard/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/standard/package-info.java@@ -16,17 +16,16 @@*//**- * Fast, general-purpose grammar-based tokenizer {@link org.apache.lucene.analysis.standard.StandardTokenizer}- * implements the Word Break rules from the Unicode Text Segmentation algorithm, as specified in- * <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.- * Unlike <code>UAX29URLEmailTokenizer</code> from the analysis module, URLs and email addresses are- * <b>not</b> tokenized as single tokens, but are instead split up into- * tokens according to the UAX#29 word break rules.- * <br>- * {@link org.apache.lucene.analysis.standard.StandardAnalyzer StandardAnalyzer} includes- * {@link org.apache.lucene.analysis.standard.StandardTokenizer StandardTokenizer},- * {@link org.apache.lucene.analysis.LowerCaseFilter LowerCaseFilter}- * and {@link org.apache.lucene.analysis.StopFilter StopFilter}.+ * Fast, general-purpose grammar-based tokenizer {@link+ * org.apache.lucene.analysis.standard.StandardTokenizer} implements the Word Break rules from the+ * Unicode Text Segmentation algorithm, as specified in <a+ * href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>. Unlike <code>+ * UAX29URLEmailTokenizer</code> from the analysis module, URLs and email addresses are <b>not</b>+ * tokenized as single tokens, but are instead split up into tokens according to the UAX#29 word+ * break rules. <br>+ * {@link org.apache.lucene.analysis.standard.StandardAnalyzer StandardAnalyzer} includes {@link+ * org.apache.lucene.analysis.standard.StandardTokenizer StandardTokenizer}, {@link+ * org.apache.lucene.analysis.LowerCaseFilter LowerCaseFilter} and {@link+ * org.apache.lucene.analysis.StopFilter StopFilter}.*/-package org.apache.lucene.analysis.standard;diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.javaindex 4c650865d35..227aa68574a 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttribute.java@@ -16,16 +16,15 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.BytesRef;/**- * This attribute can be used if you have the raw term bytes to be indexed.- * It can be used as replacement for {@link CharTermAttribute}, if binary- * terms should be indexed.+ * This attribute can be used if you have the raw term bytes to be indexed. It can be used as+ * replacement for {@link CharTermAttribute}, if binary terms should be indexed.+ ** @lucene.internal*/public interface BytesTermAttribute extends TermToBytesRefAttribute {/** Sets the {@link BytesRef} of the term */public void setBytesRef(BytesRef bytes);-}\ No newline at end of file+}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.javaindex 6cebb9d7542..8783bbfedf9 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java@@ -16,17 +16,18 @@*/package org.apache.lucene.analysis.tokenattributes;-import java.util.Objects;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;import org.apache.lucene.util.BytesRef;-/** Implementation class for {@link BytesTermAttribute}.+/**+ * Implementation class for {@link BytesTermAttribute}.+ ** @lucene.internal*/-public class BytesTermAttributeImpl extends AttributeImpl implements BytesTermAttribute, TermToBytesRefAttribute {+public class BytesTermAttributeImpl extends AttributeImpl+ implements BytesTermAttribute, TermToBytesRefAttribute {private BytesRef bytes;/** Initialize this attribute with no bytes. */@@ -55,7 +56,7 @@ public class BytesTermAttributeImpl extends AttributeImpl implements BytesTermAt@Overridepublic AttributeImpl clone() {- BytesTermAttributeImpl c = (BytesTermAttributeImpl)super.clone();+ BytesTermAttributeImpl c = (BytesTermAttributeImpl) super.clone();copyTo(c);return c;}@@ -77,4 +78,4 @@ public class BytesTermAttributeImpl extends AttributeImpl implements BytesTermAtpublic int hashCode() {return Objects.hash(bytes);}-}\ No newline at end of file+}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.javaindex 5e10bd798bb..f018ed5da23 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.java@@ -16,83 +16,88 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;-/**- * The term text of a Token.- */+/** The term text of a Token. */public interface CharTermAttribute extends Attribute, CharSequence, Appendable {-- /** Copies the contents of buffer, starting at offset for- * length characters, into the termBuffer array.- * @param buffer the buffer to copy- * @param offset the index in the buffer of the first character to copy- * @param length the number of characters to copy++ /**+ * Copies the contents of buffer, starting at offset for length characters, into the termBuffer+ * array.+ *+ * @param buffer the buffer to copy+ * @param offset the index in the buffer of the first character to copy+ * @param length the number of characters to copy*/public void copyBuffer(char[] buffer, int offset, int length);-- /** Returns the internal termBuffer character array which- * you can then directly alter. If the array is too- * small for your token, use {@link- * #resizeBuffer(int)} to increase it. After- * altering the buffer be sure to call {@link- * #setLength} to record the number of valid- * characters that were placed into the termBuffer.- * <p>- * <b>NOTE</b>: The returned buffer may be larger than- * the valid {@link #length()}.++ /**+ * Returns the internal termBuffer character array which you can then directly alter. If the array+ * is too small for your token, use {@link #resizeBuffer(int)} to increase it. After altering the+ * buffer be sure to call {@link #setLength} to record the number of valid characters that were+ * placed into the termBuffer.+ *+ * <p><b>NOTE</b>: The returned buffer may be larger than the valid {@link #length()}.*/public char[] buffer();- /** Grows the termBuffer to at least size newSize, preserving the- * existing content.- * @param newSize minimum size of the new termBuffer- * @return newly created termBuffer with {@code length >= newSize}+ /**+ * Grows the termBuffer to at least size newSize, preserving the existing content.+ *+ * @param newSize minimum size of the new termBuffer+ * @return newly created termBuffer with {@code length >= newSize}*/public char[] resizeBuffer(int newSize);- /** Set number of valid characters (length of the term) in- * the termBuffer array. Use this to truncate the termBuffer- * or to synchronize with external manipulation of the termBuffer.- * Note: to grow the size of the array,- * use {@link #resizeBuffer(int)} first.- * @param length the truncated length+ /**+ * Set number of valid characters (length of the term) in the termBuffer array. Use this to+ * truncate the termBuffer or to synchronize with external manipulation of the termBuffer. Note:+ * to grow the size of the array, use {@link #resizeBuffer(int)} first.+ *+ * @param length the truncated length*/public CharTermAttribute setLength(int length);-- /** Sets the length of the termBuffer to zero.- * Use this method before appending contents- * using the {@link Appendable} interface.++ /**+ * Sets the length of the termBuffer to zero. Use this method before appending contents using the+ * {@link Appendable} interface.*/public CharTermAttribute setEmpty();-+// the following methods are redefined to get rid of IOException declaration:@Overridepublic CharTermAttribute append(CharSequence csq);+@Overridepublic CharTermAttribute append(CharSequence csq, int start, int end);+@Overridepublic CharTermAttribute append(char c);- /** Appends the specified {@code String} to this character sequence.- * <p>The characters of the {@code String} argument are appended, in order, increasing the length of- * this sequence by the length of the argument. If argument is {@code null}, then the four- * characters {@code "null"} are appended.+ /**+ * Appends the specified {@code String} to this character sequence.+ *+ * <p>The characters of the {@code String} argument are appended, in order, increasing the length+ * of this sequence by the length of the argument. If argument is {@code null}, then the four+ * characters {@code "null"} are appended.*/public CharTermAttribute append(String s);- /** Appends the specified {@code StringBuilder} to this character sequence.- * <p>The characters of the {@code StringBuilder} argument are appended, in order, increasing the length of- * this sequence by the length of the argument. If argument is {@code null}, then the four- * characters {@code "null"} are appended.+ /**+ * Appends the specified {@code StringBuilder} to this character sequence.+ *+ * <p>The characters of the {@code StringBuilder} argument are appended, in order, increasing the+ * length of this sequence by the length of the argument. If argument is {@code null}, then the+ * four characters {@code "null"} are appended.*/public CharTermAttribute append(StringBuilder sb);- /** Appends the contents of the other {@code CharTermAttribute} to this character sequence.- * <p>The characters of the {@code CharTermAttribute} argument are appended, in order, increasing the length of- * this sequence by the length of the argument. If argument is {@code null}, then the four- * characters {@code "null"} are appended.+ /**+ * Appends the contents of the other {@code CharTermAttribute} to this character sequence.+ *+ * <p>The characters of the {@code CharTermAttribute} argument are appended, in order, increasing+ * the length of this sequence by the length of the argument. If argument is {@code null}, then+ * the four characters {@code "null"} are appended.*/public CharTermAttribute append(CharTermAttribute termAtt);}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.javaindex 9d6166d7882..bae5c60968e 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.tokenattributes;import java.nio.CharBuffer;import java.util.Objects;-import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;@@ -26,15 +25,19 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;/** Default implementation of {@link CharTermAttribute}. */-public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttribute, TermToBytesRefAttribute, Cloneable {+public class CharTermAttributeImpl extends AttributeImpl+ implements CharTermAttribute, TermToBytesRefAttribute, Cloneable {private static int MIN_BUFFER_SIZE = 10;-+private char[] termBuffer = new char[ArrayUtil.oversize(MIN_BUFFER_SIZE, Character.BYTES)];private int termLength = 0;-- /** May be used by subclasses to convert to different charsets / encodings for implementing {@link #getBytesRef()}. */++ /**+ * May be used by subclasses to convert to different charsets / encodings for implementing {@link+ * #getBytesRef()}.+ */protected BytesRefBuilder builder = new BytesRefBuilder();-+/** Initialize this attribute with empty term text */public CharTermAttributeImpl() {}@@ -49,21 +52,21 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrpublic final char[] buffer() {return termBuffer;}-+@Overridepublic final char[] resizeBuffer(int newSize) {- if(termBuffer.length < newSize){+ if (termBuffer.length < newSize) {// Not big enough; create a new array with slight// over allocation and preserve contentfinal char[] newCharBuffer = new char[ArrayUtil.oversize(newSize, Character.BYTES)];System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);termBuffer = newCharBuffer;}- return termBuffer;+ return termBuffer;}-+private void growTermBuffer(int newSize) {- if(termBuffer.length < newSize){+ if (termBuffer.length < newSize) {// Not big enough; create a new array with slight// over allocation:termBuffer = new char[ArrayUtil.oversize(newSize, Character.BYTES)];@@ -76,13 +79,13 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrtermLength = length;return this;}-+@Overridepublic final CharTermAttribute setEmpty() {termLength = 0;return this;}-+// *** TermToBytesRefAttribute interface ***@Override@@ -90,43 +93,42 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrbuilder.copyChars(termBuffer, 0, termLength);return builder.get();}-+// *** CharSequence interface ***@Overridepublic final int length() {return termLength;}-+@Overridepublic final char charAt(int index) {Objects.checkIndex(index, termLength);return termBuffer[index];}-+@Overridepublic final CharSequence subSequence(final int start, final int end) {Objects.checkFromToIndex(start, end, termLength);return new String(termBuffer, start, end - start);}-+// *** Appendable interface ***@Overridepublic final CharTermAttribute append(CharSequence csq) {if (csq == null) // needed for Appendable compliance- return appendNull();+ return appendNull();return append(csq, 0, csq.length());}-+@Overridepublic final CharTermAttribute append(CharSequence csq, int start, int end) {if (csq == null) // needed for Appendable compliance- csq = "null";+ csq = "null";// TODO: the optimized cases (jdk methods) will already do such checks, maybe re-organize this?Objects.checkFromToIndex(start, end, csq.length());final int len = end - start;- if (len == 0)- return this;+ if (len == 0) return this;resizeBuffer(termLength + len);if (len > 4) { // only use instanceof check series for longer CSQs, else simply iterateif (csq instanceof String) {@@ -137,56 +139,55 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrSystem.arraycopy(((CharTermAttribute) csq).buffer(), start, termBuffer, termLength, len);} else if (csq instanceof CharBuffer && ((CharBuffer) csq).hasArray()) {final CharBuffer cb = (CharBuffer) csq;- System.arraycopy(cb.array(), cb.arrayOffset() + cb.position() + start, termBuffer, termLength, len);+ System.arraycopy(+ cb.array(), cb.arrayOffset() + cb.position() + start, termBuffer, termLength, len);} else if (csq instanceof StringBuffer) {((StringBuffer) csq).getChars(start, end, termBuffer, termLength);} else {- while (start < end)- termBuffer[termLength++] = csq.charAt(start++);+ while (start < end) termBuffer[termLength++] = csq.charAt(start++);// no fall-through here, as termLength is updated!return this;}termLength += len;return this;} else {- while (start < end)- termBuffer[termLength++] = csq.charAt(start++);+ while (start < end) termBuffer[termLength++] = csq.charAt(start++);return this;}}-+@Overridepublic final CharTermAttribute append(char c) {resizeBuffer(termLength + 1)[termLength++] = c;return this;}-+// *** For performance some convenience methods in addition to CSQ's ***-+@Overridepublic final CharTermAttribute append(String s) {if (s == null) // needed for Appendable compliance- return appendNull();+ return appendNull();final int len = s.length();s.getChars(0, len, resizeBuffer(termLength + len), termLength);termLength += len;return this;}-+@Overridepublic final CharTermAttribute append(StringBuilder s) {if (s == null) // needed for Appendable compliance- return appendNull();+ return appendNull();final int len = s.length();s.getChars(0, len, resizeBuffer(termLength + len), termLength);termLength += len;return this;}-+@Overridepublic final CharTermAttribute append(CharTermAttribute ta) {if (ta == null) // needed for Appendable compliance- return appendNull();+ return appendNull();final int len = ta.length();System.arraycopy(ta.buffer(), 0, resizeBuffer(termLength + len), termLength, len);termLength += len;@@ -201,7 +202,7 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrtermBuffer[termLength++] = 'l';return this;}-+// *** AttributeImpl ***@Override@@ -213,12 +214,12 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttr@Overridepublic void clear() {- termLength = 0;+ termLength = 0;}@Overridepublic CharTermAttributeImpl clone() {- CharTermAttributeImpl t = (CharTermAttributeImpl)super.clone();+ CharTermAttributeImpl t = (CharTermAttributeImpl) super.clone();// Do a deep clonet.termBuffer = new char[this.termLength];System.arraycopy(this.termBuffer, 0, t.termBuffer, 0, this.termLength);@@ -226,47 +227,42 @@ public class CharTermAttributeImpl extends AttributeImpl implements CharTermAttrt.builder.copyBytes(builder.get());return t;}-+@Overridepublic boolean equals(Object other) {if (other == this) {return true;}-+if (other instanceof CharTermAttributeImpl) {final CharTermAttributeImpl o = ((CharTermAttributeImpl) other);- if (termLength != o.termLength)- return false;- for(int i=0;i<termLength;i++) {+ if (termLength != o.termLength) return false;+ for (int i = 0; i < termLength; i++) {if (termBuffer[i] != o.termBuffer[i]) {return false;}}return true;}-+return false;}- /**- * Returns solely the term text as specified by the- * {@link CharSequence} interface.- */+ /** Returns solely the term text as specified by the {@link CharSequence} interface. */@Overridepublic String toString() {return new String(termBuffer, 0, termLength);}-+@Overridepublic void reflectWith(AttributeReflector reflector) {reflector.reflect(CharTermAttribute.class, "term", toString());reflector.reflect(TermToBytesRefAttribute.class, "bytes", getBytesRef());}-+@Overridepublic void copyTo(AttributeImpl target) {CharTermAttribute t = (CharTermAttribute) target;t.copyBuffer(termBuffer, 0, termLength);}-}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.javaindex 842e47d06ff..e30f807d60c 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java@@ -16,22 +16,23 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.util.Attribute;/**- * This attribute can be used to pass different flags down the {@link Tokenizer} chain,- * e.g. from one TokenFilter to another one.- * <p>- * This is completely distinct from {@link TypeAttribute}, although they do share similar purposes.- * The flags can be used to encode information about the token for use by other- * {@link org.apache.lucene.analysis.TokenFilter}s.+ * This attribute can be used to pass different flags down the {@link Tokenizer} chain, e.g. from+ * one TokenFilter to another one.+ *+ * <p>This is completely distinct from {@link TypeAttribute}, although they do share similar+ * purposes. The flags can be used to encode information about the token for use by other {@link+ * org.apache.lucene.analysis.TokenFilter}s.+ ** @lucene.experimental While we think this is here to stay, we may want to change it to be a long.*/public interface FlagsAttribute extends Attribute {/**- * Get the bitset for any bits that have been set.+ * Get the bitset for any bits that have been set.+ ** @return The bits* @see #getFlags()*/@@ -39,7 +40,8 @@ public interface FlagsAttribute extends Attribute {/*** Set the flags to a new bitset.+ ** @see #getFlags()*/- public void setFlags(int flags);+ public void setFlags(int flags);}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.javaindex 208a32d2b20..9a1bdc37db1 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttributeImpl.java@@ -16,17 +16,16 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link FlagsAttribute}. */public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute, Cloneable {private int flags = 0;-+/** Initialize this attribute with no bits set */public FlagsAttributeImpl() {}-+@Overridepublic int getFlags() {return flags;@@ -36,7 +35,7 @@ public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute,public void setFlags(int flags) {this.flags = flags;}-+@Overridepublic void clear() {flags = 0;@@ -47,11 +46,11 @@ public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute,if (this == other) {return true;}-+if (other instanceof FlagsAttributeImpl) {return ((FlagsAttributeImpl) other).flags == flags;}-+return false;}@@ -59,7 +58,7 @@ public class FlagsAttributeImpl extends AttributeImpl implements FlagsAttribute,public int hashCode() {return flags;}-+@Overridepublic void copyTo(AttributeImpl target) {FlagsAttribute t = (FlagsAttribute) target;diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.javaindex 8b99bc2eb8c..0cee55b0b00 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttribute.java@@ -16,35 +16,30 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.util.Attribute;/**- * This attribute can be used to mark a token as a keyword. Keyword aware- * {@link TokenStream}s can decide to modify a token based on the return value- * of {@link #isKeyword()} if the token is modified. Stemming filters for- * instance can use this attribute to conditionally skip a term if+ * This attribute can be used to mark a token as a keyword. Keyword aware {@link TokenStream}s can+ * decide to modify a token based on the return value of {@link #isKeyword()} if the token is+ * modified. Stemming filters for instance can use this attribute to conditionally skip a term if* {@link #isKeyword()} returns <code>true</code>.*/public interface KeywordAttribute extends Attribute {/**- * Returns <code>true</code> if the current token is a keyword, otherwise- * <code>false</code>- *- * @return <code>true</code> if the current token is a keyword, otherwise- * <code>false</code>+ * Returns <code>true</code> if the current token is a keyword, otherwise <code>false</code>+ *+ * @return <code>true</code> if the current token is a keyword, otherwise <code>false</code>* @see #setKeyword(boolean)*/public boolean isKeyword();/*** Marks the current token as keyword if set to <code>true</code>.- *- * @param isKeyword- * <code>true</code> if the current token is a keyword, otherwise- * <code>false</code>.+ *+ * @param isKeyword <code>true</code> if the current token is a keyword, otherwise <code>false+ * </code>.* @see #isKeyword()*/public void setKeyword(boolean isKeyword);diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.javaindex bda62bbf34c..ae0dbaa06ed 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/KeywordAttributeImpl.java@@ -16,15 +16,13 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link KeywordAttribute}. */-public final class KeywordAttributeImpl extends AttributeImpl implements- KeywordAttribute {+public final class KeywordAttributeImpl extends AttributeImpl implements KeywordAttribute {private boolean keyword;-+/** Initialize this attribute with the keyword value as false. */public KeywordAttributeImpl() {}@@ -46,10 +44,8 @@ public final class KeywordAttributeImpl extends AttributeImpl implements@Overridepublic boolean equals(Object obj) {- if (this == obj)- return true;- if (getClass() != obj.getClass())- return false;+ if (this == obj) return true;+ if (getClass() != obj.getClass()) return false;final KeywordAttributeImpl other = (KeywordAttributeImpl) obj;return keyword == other.keyword;}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.javaindex 1153448cd2a..0c68459c2b2 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java@@ -16,40 +16,37 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;-/**- * The start and end character offset of a Token.- */+/** The start and end character offset of a Token. */public interface OffsetAttribute extends Attribute {- /**- * Returns this Token's starting offset, the position of the first character- * corresponding to this token in the source text.- * <p>- * Note that the difference between {@link #endOffset()} and <code>startOffset()</code>- * may not be equal to termText.length(), as the term text may have been altered by a- * stemmer or some other filter.- * @see #setOffset(int, int)+ /**+ * Returns this Token's starting offset, the position of the first character corresponding to this+ * token in the source text.+ *+ * <p>Note that the difference between {@link #endOffset()} and <code>startOffset()</code> may not+ * be equal to termText.length(), as the term text may have been altered by a stemmer or some+ * other filter.+ *+ * @see #setOffset(int, int)*/public int startOffset();-- /**+ /*** Set the starting and ending offset.- * @throws IllegalArgumentException If <code>startOffset</code> or <code>endOffset</code>- * are negative, or if <code>startOffset</code> is greater than- * <code>endOffset</code>+ *+ * @throws IllegalArgumentException If <code>startOffset</code> or <code>endOffset</code> are+ * negative, or if <code>startOffset</code> is greater than <code>endOffset</code>* @see #startOffset()* @see #endOffset()*/public void setOffset(int startOffset, int endOffset);-- /**- * Returns this Token's ending offset, one greater than the position of the- * last character corresponding to this token in the source text. The length- * of the token in the source text is (<code>endOffset()</code> - {@link #startOffset()}).+ /**+ * Returns this Token's ending offset, one greater than the position of the last character+ * corresponding to this token in the source text. The length of the token in the source text is (+ * <code>endOffset()</code> - {@link #startOffset()}).+ ** @see #setOffset(int, int)*/public int endOffset();diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.javaindex 166d6b21607..8ddae001100 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttributeImpl.java@@ -16,7 +16,6 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;@@ -24,7 +23,7 @@ import org.apache.lucene.util.AttributeReflector;public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttribute, Cloneable {private int startOffset;private int endOffset;-+/** Initialize this attribute with startOffset and endOffset of 0. */public OffsetAttributeImpl() {}@@ -43,20 +42,23 @@ public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttribut// OffsetAttif (startOffset < 0 || endOffset < startOffset) {- throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset; got "- + "startOffset=" + startOffset + ",endOffset=" + endOffset);+ throw new IllegalArgumentException(+ "startOffset must be non-negative, and endOffset must be >= startOffset; got "+ + "startOffset="+ + startOffset+ + ",endOffset="+ + endOffset);}this.startOffset = startOffset;this.endOffset = endOffset;}-+@Overridepublic int endOffset() {return endOffset;}-@Overridepublic void clear() {// TODO: we could use -1 as default here? Then we can@@ -64,18 +66,18 @@ public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttributstartOffset = 0;endOffset = 0;}-+@Overridepublic boolean equals(Object other) {if (other == this) {return true;}-+if (other instanceof OffsetAttributeImpl) {OffsetAttributeImpl o = (OffsetAttributeImpl) other;return o.startOffset == startOffset && o.endOffset == endOffset;}-+return false;}@@ -84,13 +86,13 @@ public class OffsetAttributeImpl extends AttributeImpl implements OffsetAttributint code = startOffset;code = code * 31 + endOffset;return code;- }-+ }+@Overridepublic void copyTo(AttributeImpl target) {OffsetAttribute t = (OffsetAttribute) target;t.setOffset(startOffset, endOffset);- }+ }@Overridepublic void reflectWith(AttributeReflector reflector) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.javaindex d0573972c5c..296af82b4b4 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java@@ -16,35 +16,40 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;-/** Default implementation of the common attributes used by Lucene:<ul>- * <li>{@link CharTermAttribute}- * <li>{@link TypeAttribute}- * <li>{@link PositionIncrementAttribute}- * <li>{@link PositionLengthAttribute}- * <li>{@link OffsetAttribute}- * <li>{@link TermFrequencyAttribute}- * </ul>*/-public class PackedTokenAttributeImpl extends CharTermAttributeImpl- implements TypeAttribute, PositionIncrementAttribute,- PositionLengthAttribute, OffsetAttribute,- TermFrequencyAttribute {-- private int startOffset,endOffset;+/**+ * Default implementation of the common attributes used by Lucene:+ *+ * <ul>+ * <li>{@link CharTermAttribute}+ * <li>{@link TypeAttribute}+ * <li>{@link PositionIncrementAttribute}+ * <li>{@link PositionLengthAttribute}+ * <li>{@link OffsetAttribute}+ * <li>{@link TermFrequencyAttribute}+ * </ul>+ */+public class PackedTokenAttributeImpl extends CharTermAttributeImpl+ implements TypeAttribute,+ PositionIncrementAttribute,+ PositionLengthAttribute,+ OffsetAttribute,+ TermFrequencyAttribute {++ private int startOffset, endOffset;private String type = DEFAULT_TYPE;private int positionIncrement = 1;private int positionLength = 1;private int termFrequency = 1;/** Constructs the attribute implementation. */- public PackedTokenAttributeImpl() {- }+ public PackedTokenAttributeImpl() {}/*** {@inheritDoc}+ ** @see PositionIncrementAttribute*/@Override@@ -57,6 +62,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see PositionIncrementAttribute*/@Override@@ -66,18 +72,21 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see PositionLengthAttribute*/@Overridepublic void setPositionLength(int positionLength) {if (positionLength < 1) {- throw new IllegalArgumentException("Position length must be 1 or greater: got " + positionLength);+ throw new IllegalArgumentException(+ "Position length must be 1 or greater: got " + positionLength);}this.positionLength = positionLength;}/*** {@inheritDoc}+ ** @see PositionLengthAttribute*/@Override@@ -87,6 +96,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see OffsetAttribute*/@Override@@ -96,6 +106,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see OffsetAttribute*/@Override@@ -105,13 +116,18 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see OffsetAttribute*/@Overridepublic void setOffset(int startOffset, int endOffset) {if (startOffset < 0 || endOffset < startOffset) {- throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset; got "- + "startOffset=" + startOffset + ",endOffset=" + endOffset);+ throw new IllegalArgumentException(+ "startOffset must be non-negative, and endOffset must be >= startOffset; got "+ + "startOffset="+ + startOffset+ + ",endOffset="+ + endOffset);}this.startOffset = startOffset;this.endOffset = endOffset;@@ -119,6 +135,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see TypeAttribute*/@Override@@ -128,6 +145,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl/*** {@inheritDoc}+ ** @see TypeAttribute*/@Override@@ -138,7 +156,8 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl@Overridepublic final void setTermFrequency(int termFrequency) {if (termFrequency < 1) {- throw new IllegalArgumentException("Term frequency must be 1 or greater; got " + termFrequency);+ throw new IllegalArgumentException(+ "Term frequency must be 1 or greater; got " + termFrequency);}this.termFrequency = termFrequency;}@@ -148,8 +167,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImplreturn termFrequency;}- /** Resets the attributes- */+ /** Resets the attributes */@Overridepublic void clear() {super.clear();@@ -158,9 +176,8 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImplstartOffset = endOffset = 0;type = DEFAULT_TYPE;}-- /** Resets the attributes at end- */++ /** Resets the attributes at end */@Overridepublic void end() {super.end();@@ -175,21 +192,18 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl@Overridepublic boolean equals(Object obj) {- if (obj == this)- return true;+ if (obj == this) return true;if (obj instanceof PackedTokenAttributeImpl) {final PackedTokenAttributeImpl other = (PackedTokenAttributeImpl) obj;- return (startOffset == other.startOffset &&- endOffset == other.endOffset &&- positionIncrement == other.positionIncrement &&- positionLength == other.positionLength &&- (type == null ? other.type == null : type.equals(other.type)) &&- termFrequency == other.termFrequency &&- super.equals(obj)- );- } else- return false;+ return (startOffset == other.startOffset+ && endOffset == other.endOffset+ && positionIncrement == other.positionIncrement+ && positionLength == other.positionLength+ && (type == null ? other.type == null : type.equals(other.type))+ && termFrequency == other.termFrequency+ && super.equals(obj));+ } else return false;}@Override@@ -199,8 +213,7 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImplcode = code * 31 + endOffset;code = code * 31 + positionIncrement;code = code * 31 + positionLength;- if (type != null)- code = code * 31 + type.hashCode();+ if (type != null) code = code * 31 + type.hashCode();code = code * 31 + termFrequency;return code;}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.javaindex 73ecdebb5e1..2c45dd64107 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java@@ -16,31 +16,32 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;import org.apache.lucene.util.BytesRef;/*** The payload of a Token.- * <p>- * The payload is stored in the index at each position, and can- * be used to influence scoring when using Payload-based queries.- * <p>- * NOTE: because the payload will be stored at each position, it's usually- * best to use the minimum number of bytes necessary. Some codec implementations- * may optimize payload storage when all payloads have the same length.- *+ *+ * <p>The payload is stored in the index at each position, and can be used to influence scoring when+ * using Payload-based queries.+ *+ * <p>NOTE: because the payload will be stored at each position, it's usually best to use the+ * minimum number of bytes necessary. Some codec implementations may optimize payload storage when+ * all payloads have the same length.+ ** @see org.apache.lucene.index.PostingsEnum*/public interface PayloadAttribute extends Attribute {/*** Returns this Token's payload.+ ** @see #setPayload(BytesRef)- */+ */public BytesRef getPayload();- /**+ /*** Sets this Token's payload.+ ** @see #getPayload()*/public void setPayload(BytesRef payload);diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.javaindex 5eaed0cac78..da36ebd01f9 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttributeImpl.java@@ -16,27 +16,22 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;import org.apache.lucene.util.BytesRef;/** Default implementation of {@link PayloadAttribute}. */public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribute, Cloneable {- private BytesRef payload;-- /**- * Initialize this attribute with no payload.- */+ private BytesRef payload;++ /** Initialize this attribute with no payload. */public PayloadAttributeImpl() {}-- /**- * Initialize this attribute with the given payload.- */++ /** Initialize this attribute with the given payload. */public PayloadAttributeImpl(BytesRef payload) {this.payload = payload;}-+@Overridepublic BytesRef getPayload() {return this.payload;@@ -46,14 +41,14 @@ public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribpublic void setPayload(BytesRef payload) {this.payload = payload;}-+@Overridepublic void clear() {payload = null;}@Override- public PayloadAttributeImpl clone() {+ public PayloadAttributeImpl clone() {PayloadAttributeImpl clone = (PayloadAttributeImpl) super.clone();if (payload != null) {clone.payload = BytesRef.deepCopyOf(payload);@@ -66,16 +61,16 @@ public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribif (other == this) {return true;}-+if (other instanceof PayloadAttribute) {PayloadAttributeImpl o = (PayloadAttributeImpl) other;if (o.payload == null || payload == null) {return o.payload == null && payload == null;}-+return o.payload.equals(payload);}-+return false;}@@ -88,7 +83,7 @@ public class PayloadAttributeImpl extends AttributeImpl implements PayloadAttribpublic void copyTo(AttributeImpl target) {PayloadAttribute t = (PayloadAttribute) target;t.setPayload((payload == null) ? null : BytesRef.deepCopyOf(payload));- }+ }@Overridepublic void reflectWith(AttributeReflector reflector) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.javaindex 2e73713be8c..90c571eba82 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java@@ -16,46 +16,44 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;-/** Determines the position of this token- * relative to the previous Token in a TokenStream, used in phrase- * searching.+/**+ * Determines the position of this token relative to the previous Token in a TokenStream, used in+ * phrase searching.** <p>The default value is one.*- * <p>Some common uses for this are:<ul>- *- * <li>Set it to zero to put multiple terms in the same position. This is- * useful if, e.g., a word has multiple stems. Searches for phrases- * including either stem will match. In this case, all but the first stem's- * increment should be set to zero: the increment of the first instance- * should be one. Repeating a token with an increment of zero can also be- * used to boost the scores of matches on that token.- *- * <li>Set it to values greater than one to inhibit exact phrase matches.- * If, for example, one does not want phrases to match across removed stop- * words, then one could build a stop word filter that removes stop words and- * also sets the increment to the number of stop words removed before each- * non-stop word. Then exact phrase queries will only match when the terms- * occur with no intervening stop words.+ * <p>Some common uses for this are:*+ * <ul>+ * <li>Set it to zero to put multiple terms in the same position. This is useful if, e.g., a word+ * has multiple stems. Searches for phrases including either stem will match. In this case,+ * all but the first stem's increment should be set to zero: the increment of the first+ * instance should be one. Repeating a token with an increment of zero can also be used to+ * boost the scores of matches on that token.+ * <li>Set it to values greater than one to inhibit exact phrase matches. If, for example, one+ * does not want phrases to match across removed stop words, then one could build a stop word+ * filter that removes stop words and also sets the increment to the number of stop words+ * removed before each non-stop word. Then exact phrase queries will only match when the terms+ * occur with no intervening stop words.* </ul>- *+ ** @see org.apache.lucene.index.PostingsEnum*/public interface PositionIncrementAttribute extends Attribute {- /** Set the position increment. The default value is one.+ /**+ * Set the position increment. The default value is one.** @param positionIncrement the distance from the prior term- * @throws IllegalArgumentException if <code>positionIncrement</code>- * is negative.+ * @throws IllegalArgumentException if <code>positionIncrement</code> is negative.* @see #getPositionIncrement()*/public void setPositionIncrement(int positionIncrement);- /** Returns the position increment of this Token.+ /**+ * Returns the position increment of this Token.+ ** @see #setPositionIncrement(int)*/public int getPositionIncrement();diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.javaindex e89fec12715..80ef0d42fbf 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java@@ -16,21 +16,22 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link PositionIncrementAttribute}. */-public class PositionIncrementAttributeImpl extends AttributeImpl implements PositionIncrementAttribute, Cloneable {+public class PositionIncrementAttributeImpl extends AttributeImpl+ implements PositionIncrementAttribute, Cloneable {private int positionIncrement = 1;-+/** Initialize this attribute with position increment of 1 */public PositionIncrementAttributeImpl() {}@Overridepublic void setPositionIncrement(int positionIncrement) {if (positionIncrement < 0) {- throw new IllegalArgumentException("Position increment must be zero or greater; got " + positionIncrement);+ throw new IllegalArgumentException(+ "Position increment must be zero or greater; got " + positionIncrement);}this.positionIncrement = positionIncrement;}@@ -44,7 +45,7 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Pospublic void clear() {this.positionIncrement = 1;}-+@Overridepublic void end() {this.positionIncrement = 0;@@ -55,12 +56,12 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Posif (other == this) {return true;}-+if (other instanceof PositionIncrementAttributeImpl) {PositionIncrementAttributeImpl _other = (PositionIncrementAttributeImpl) other;- return positionIncrement == _other.positionIncrement;+ return positionIncrement == _other.positionIncrement;}-+return false;}@@ -68,12 +69,12 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Pospublic int hashCode() {return positionIncrement;}-+@Overridepublic void copyTo(AttributeImpl target) {PositionIncrementAttribute t = (PositionIncrementAttribute) target;t.setPositionIncrement(positionIncrement);- }+ }@Overridepublic void reflectWith(AttributeReflector reflector) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.javaindex 210eb0fcebc..ab4996a5439 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttribute.java@@ -16,35 +16,31 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;-/** Determines how many positions this- * token spans. Very few analyzer components actually- * produce this attribute, and indexing ignores it, but- * it's useful to express the graph structure naturally- * produced by decompounding, word splitting/joining,- * synonym filtering, etc.+/**+ * Determines how many positions this token spans. Very few analyzer components actually produce+ * this attribute, and indexing ignores it, but it's useful to express the graph structure naturally+ * produced by decompounding, word splitting/joining, synonym filtering, etc.*- * <p>NOTE: this is optional, and most analyzers- * don't change the default value (1). */-+ * <p>NOTE: this is optional, and most analyzers don't change the default value (1).+ */public interface PositionLengthAttribute extends Attribute {/*** Set the position length of this Token.- * <p>- * The default value is one.- * @param positionLength how many positions this token- * spans.- * @throws IllegalArgumentException if <code>positionLength</code>- * is zero or negative.+ *+ * <p>The default value is one.+ *+ * @param positionLength how many positions this token spans.+ * @throws IllegalArgumentException if <code>positionLength</code> is zero or negative.* @see #getPositionLength()*/public void setPositionLength(int positionLength);- /** Returns the position length of this Token.+ /**+ * Returns the position length of this Token.+ ** @see #setPositionLength*/public int getPositionLength();}-diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.javaindex d019a2b6d61..3a59faa60e0 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionLengthAttributeImpl.java@@ -16,21 +16,22 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link PositionLengthAttribute}. */-public class PositionLengthAttributeImpl extends AttributeImpl implements PositionLengthAttribute, Cloneable {+public class PositionLengthAttributeImpl extends AttributeImpl+ implements PositionLengthAttribute, Cloneable {private int positionLength = 1;-+/** Initializes this attribute with position length of 1. */public PositionLengthAttributeImpl() {}-+@Overridepublic void setPositionLength(int positionLength) {if (positionLength < 1) {- throw new IllegalArgumentException("Position length must be 1 or greater; got " + positionLength);+ throw new IllegalArgumentException(+ "Position length must be 1 or greater; got " + positionLength);}this.positionLength = positionLength;}@@ -44,18 +45,18 @@ public class PositionLengthAttributeImpl extends AttributeImpl implements Positipublic void clear() {this.positionLength = 1;}-+@Overridepublic boolean equals(Object other) {if (other == this) {return true;}-+if (other instanceof PositionLengthAttributeImpl) {PositionLengthAttributeImpl _other = (PositionLengthAttributeImpl) other;- return positionLength == _other.positionLength;+ return positionLength == _other.positionLength;}-+return false;}@@ -63,12 +64,12 @@ public class PositionLengthAttributeImpl extends AttributeImpl implements Positipublic int hashCode() {return positionLength;}-+@Overridepublic void copyTo(AttributeImpl target) {PositionLengthAttribute t = (PositionLengthAttribute) target;t.setPositionLength(positionLength);- }+ }@Overridepublic void reflectWith(AttributeReflector reflector) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.javaindex 0f33aef69a9..bad431c7684 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttribute.java@@ -20,9 +20,11 @@ package org.apache.lucene.analysis.tokenattributes;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.util.Attribute;-/** Sets the custom term frequency of a term within one document. If this attribute- * is present in your analysis chain for a given field, that field must be indexed with- * {@link IndexOptions#DOCS_AND_FREQS}. */+/**+ * Sets the custom term frequency of a term within one document. If this attribute is present in+ * your analysis chain for a given field, that field must be indexed with {@link+ * IndexOptions#DOCS_AND_FREQS}.+ */public interface TermFrequencyAttribute extends Attribute {/** Set the custom term frequency of the current term within one document. */diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.javaindex c21445358b9..4e8520f384f 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermFrequencyAttributeImpl.java@@ -16,21 +16,22 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link TermFrequencyAttribute}. */-public class TermFrequencyAttributeImpl extends AttributeImpl implements TermFrequencyAttribute, Cloneable {+public class TermFrequencyAttributeImpl extends AttributeImpl+ implements TermFrequencyAttribute, Cloneable {private int termFrequency = 1;-+/** Initialize this attribute with term frequency of 1 */public TermFrequencyAttributeImpl() {}@Overridepublic void setTermFrequency(int termFrequency) {if (termFrequency < 1) {- throw new IllegalArgumentException("Term frequency must be 1 or greater; got " + termFrequency);+ throw new IllegalArgumentException(+ "Term frequency must be 1 or greater; got " + termFrequency);}this.termFrequency = termFrequency;}@@ -44,7 +45,7 @@ public class TermFrequencyAttributeImpl extends AttributeImpl implements TermFrepublic void clear() {this.termFrequency = 1;}-+@Overridepublic void end() {this.termFrequency = 1;@@ -55,12 +56,12 @@ public class TermFrequencyAttributeImpl extends AttributeImpl implements TermFreif (other == this) {return true;}-+if (other instanceof TermFrequencyAttributeImpl) {TermFrequencyAttributeImpl _other = (TermFrequencyAttributeImpl) other;- return termFrequency == _other.termFrequency;+ return termFrequency == _other.termFrequency;}-+return false;}@@ -68,12 +69,12 @@ public class TermFrequencyAttributeImpl extends AttributeImpl implements TermFrepublic int hashCode() {return Integer.hashCode(termFrequency);}-+@Overridepublic void copyTo(AttributeImpl target) {TermFrequencyAttribute t = (TermFrequencyAttribute) target;t.setTermFrequency(termFrequency);- }+ }@Overridepublic void reflectWith(AttributeReflector reflector) {diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.javaindex 7fe5e257d77..19b0b9fb7ef 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TermToBytesRefAttribute.java@@ -16,15 +16,15 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;import org.apache.lucene.util.BytesRef;/**- * This attribute is requested by TermsHashPerField to index the contents.- * This attribute can be used to customize the final byte[] encoding of terms.- * <p>- * Consumers of this attribute call {@link #getBytesRef()} for each term. Example:+ * This attribute is requested by TermsHashPerField to index the contents. This attribute can be+ * used to customize the final byte[] encoding of terms.+ *+ * <p>Consumers of this attribute call {@link #getBytesRef()} for each term. Example:+ ** <pre class="prettyprint">* final TermToBytesRefAttribute termAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);*@@ -32,7 +32,7 @@ import org.apache.lucene.util.BytesRef;* final BytesRef bytes = termAtt.getBytesRef();** if (isInteresting(bytes)) {- *+ ** // because the bytes are reused by the attribute (like CharTermAttribute's char[] buffer),* // you should make a copy if you need persistent access to the bytes, otherwise they will* // be rewritten across calls to incrementToken()@@ -42,15 +42,17 @@ import org.apache.lucene.util.BytesRef;* }* ...* </pre>- * @lucene.internal This is a very expert and internal API, please use- * {@link CharTermAttribute} and its implementation for UTF-8 terms; to- * index binary terms, use {@link BytesTermAttribute} and its implementation.+ *+ * @lucene.internal This is a very expert and internal API, please use {@link CharTermAttribute} and+ * its implementation for UTF-8 terms; to index binary terms, use {@link BytesTermAttribute} and+ * its implementation.*/public interface TermToBytesRefAttribute extends Attribute {-+/**- * Retrieve this attribute's BytesRef. The bytes are updated from the current term.- * The implementation may return a new instance or keep the previous one.+ * Retrieve this attribute's BytesRef. The bytes are updated from the current term. The+ * implementation may return a new instance or keep the previous one.+ ** @return a BytesRef to be indexed (only stays valid until token stream gets incremented)*/public BytesRef getBytesRef();diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.javaindex 74ceb72d3d5..89af7b3da29 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java@@ -16,26 +16,25 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.Attribute;-/**- * A Token's lexical type. The Default value is "word".- */+/** A Token's lexical type. The Default value is "word". */public interface TypeAttribute extends Attribute {/** the default type */public static final String DEFAULT_TYPE = "word";- /**- * Returns this Token's lexical type. Defaults to "word".+ /**+ * Returns this Token's lexical type. Defaults to "word".+ ** @see #setType(String)*/public String type();- /**+ /*** Set the lexical type.- * @see #type()+ *+ * @see #type()*/public void setType(String type);}diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.javaindex 7486b698f3e..dbe5e942e02 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/TypeAttributeImpl.java@@ -16,24 +16,23 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;/** Default implementation of {@link TypeAttribute}. */public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, Cloneable {private String type;-+/** Initialize this attribute with {@link TypeAttribute#DEFAULT_TYPE} */public TypeAttributeImpl() {- this(DEFAULT_TYPE);+ this(DEFAULT_TYPE);}-+/** Initialize this attribute with <code>type</code> */public TypeAttributeImpl(String type) {this.type = type;}-+@Overridepublic String type() {return type;@@ -46,7 +45,7 @@ public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, C@Overridepublic void clear() {- type = DEFAULT_TYPE;+ type = DEFAULT_TYPE;}@Override@@ -54,12 +53,12 @@ public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, Cif (other == this) {return true;}-+if (other instanceof TypeAttributeImpl) {final TypeAttributeImpl o = (TypeAttributeImpl) other;return (this.type == null ? o.type == null : this.type.equals(o.type));}-+return false;}@@ -67,7 +66,7 @@ public class TypeAttributeImpl extends AttributeImpl implements TypeAttribute, Cpublic int hashCode() {return (type == null) ? 0 : type.hashCode();}-+@Overridepublic void copyTo(AttributeImpl target) {TypeAttribute t = (TypeAttribute) target;diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/package-info.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/package-info.javaindex 7ad002982d5..a70ffcfb06e 100644--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/package-info.java@@ -15,7 +15,5 @@* limitations under the License.*/-/**- * General-purpose attributes for text analysis.- */+/** General-purpose attributes for text analysis. */package org.apache.lucene.analysis.tokenattributes;diff --git a/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java b/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.javaindex 62420957b92..6d79dc59375 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardAnalyzer.java@@ -16,7 +16,6 @@*/package org.apache.lucene.analysis.standard;-import java.io.IOException;import java.io.Reader;import java.io.StringReader;@@ -24,7 +23,6 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Random;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.MockGraphTokenFilter;@@ -36,45 +34,51 @@ import org.apache.lucene.util.TestUtil;public class TestStandardAnalyzer extends BaseTokenStreamTestCase {- // LUCENE-5897: slow tokenization of strings of the form (\p{WB:ExtendNumLet}[\p{WB:Format}\p{WB:Extend}]*)++ // LUCENE-5897: slow tokenization of strings of the form+ // (\p{WB:ExtendNumLet}[\p{WB:Format}\p{WB:Extend}]*)+@Slowpublic void testLargePartiallyMatchingToken() throws Exception {// TODO: get these lists of chars matching a property from ICU4J// http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakProperty.txt- char[] WordBreak_ExtendNumLet_chars = "_\u203f\u2040\u2054\ufe33\ufe34\ufe4d\ufe4e\ufe4f\uff3f".toCharArray();+ char[] WordBreak_ExtendNumLet_chars =+ "_\u203f\u2040\u2054\ufe33\ufe34\ufe4d\ufe4e\ufe4f\uff3f".toCharArray();// http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakProperty.txt- int[] WordBreak_Format_chars // only the first char in ranges- = { 0xAD, 0x600, 0x61C, 0x6DD, 0x70F, 0x180E, 0x200E, 0x202A, 0x2060, 0x2066, 0xFEFF,- 0xFFF9, 0x110BD, 0x1D173, 0xE0001, 0xE0020 };+ int[] WordBreak_Format_chars // only the first char in ranges+ = {+ 0xAD, 0x600, 0x61C, 0x6DD, 0x70F, 0x180E, 0x200E, 0x202A, 0x2060, 0x2066, 0xFEFF, 0xFFF9,+ 0x110BD, 0x1D173, 0xE0001, 0xE0020+ };// http://www.unicode.org/Public/6.3.0/ucd/auxiliary/WordBreakProperty.txtint[] WordBreak_Extend_chars // only the first char in ranges- = { 0x300, 0x483, 0x591, 0x5bf, 0x5c1, 0x5c4, 0x5c7, 0x610, 0x64b, 0x670, 0x6d6, 0x6df,- 0x6e7, 0x6ea, 0x711, 0x730, 0x7a6, 0x7eb, 0x816, 0x81b, 0x825, 0x829, 0x859, 0x8e4,- 0x900, 0x93a, 0x93e, 0x951, 0x962, 0x981, 0x9bc, 0x9be, 0x9c7, 0x9cb, 0x9d7, 0x9e2,- 0xa01, 0xa3c, 0xa3e, 0xa47, 0xa4b, 0xa51, 0xa70, 0xa75, 0xa81, 0xabc, 0xabe, 0xac7,- 0xacb, 0xae2, 0xb01, 0xb3c, 0xb3e, 0xb47, 0xb4b, 0xb56, 0xb62, 0xb82, 0xbbe, 0xbc6,- 0xbca, 0xbd7, 0xc01, 0xc3e, 0xc46, 0xc4a, 0xc55, 0xc62, 0xc82, 0xcbc, 0xcbe, 0xcc6,- 0xcca, 0xcd5, 0xce2, 0xd02, 0xd3e, 0xd46, 0xd4a, 0xd57, 0xd62, 0xd82, 0xdca, 0xdcf,- 0xdd6, 0xdd8, 0xdf2, 0xe31, 0xe34, 0xe47, 0xeb1, 0xeb4, 0xebb, 0xec8, 0xf18, 0xf35,- 0xf37, 0xf39, 0xf3e, 0xf71, 0xf86, 0xf8d, 0xf99, 0xfc6, 0x102b, 0x1056, 0x105e, 0x1062,- 0x1067, 0x1071, 0x1082, 0x108f, 0x109a, 0x135d, 0x1712, 0x1732, 0x1752, 0x1772, 0x17b4,- 0x17dd, 0x180b, 0x18a9, 0x1920, 0x1930, 0x19b0, 0x19c8, 0x1a17, 0x1a55, 0x1a60, 0x1a7f,- 0x1b00, 0x1b34, 0x1b6b, 0x1b80, 0x1ba1, 0x1be6, 0x1c24, 0x1cd0, 0x1cd4, 0x1ced, 0x1cf2,- 0x1dc0, 0x1dfc, 0x200c, 0x20d0, 0x2cef, 0x2d7f, 0x2de0, 0x302a, 0x3099, 0xa66f, 0xa674,- 0xa69f, 0xa6f0, 0xa802, 0xa806, 0xa80b, 0xa823, 0xa880, 0xa8b4, 0xa8e0, 0xa926, 0xa947,- 0xa980, 0xa9b3, 0xaa29, 0xaa43, 0xaa4c, 0xaa7b, 0xaab0, 0xaab2, 0xaab7, 0xaabe, 0xaac1,- 0xaaeb, 0xaaf5, 0xabe3, 0xabec, 0xfb1e, 0xfe00, 0xfe20, 0xff9e, 0x101fd, 0x10a01,- 0x10a05, 0x10a0C, 0x10a38, 0x10a3F, 0x11000, 0x11001, 0x11038, 0x11080, 0x11082,- 0x110b0, 0x110b3, 0x110b7, 0x110b9, 0x11100, 0x11127, 0x1112c, 0x11180, 0x11182,- 0x111b3, 0x111b6, 0x111bF, 0x116ab, 0x116ac, 0x116b0, 0x116b6, 0x16f51, 0x16f8f,- 0x1d165, 0x1d167, 0x1d16d, 0x1d17b, 0x1d185, 0x1d1aa, 0x1d242, 0xe0100 };-+ = {+ 0x300, 0x483, 0x591, 0x5bf, 0x5c1, 0x5c4, 0x5c7, 0x610, 0x64b, 0x670, 0x6d6, 0x6df, 0x6e7,+ 0x6ea, 0x711, 0x730, 0x7a6, 0x7eb, 0x816, 0x81b, 0x825, 0x829, 0x859, 0x8e4, 0x900, 0x93a,+ 0x93e, 0x951, 0x962, 0x981, 0x9bc, 0x9be, 0x9c7, 0x9cb, 0x9d7, 0x9e2, 0xa01, 0xa3c, 0xa3e,+ 0xa47, 0xa4b, 0xa51, 0xa70, 0xa75, 0xa81, 0xabc, 0xabe, 0xac7, 0xacb, 0xae2, 0xb01, 0xb3c,+ 0xb3e, 0xb47, 0xb4b, 0xb56, 0xb62, 0xb82, 0xbbe, 0xbc6, 0xbca, 0xbd7, 0xc01, 0xc3e, 0xc46,+ 0xc4a, 0xc55, 0xc62, 0xc82, 0xcbc, 0xcbe, 0xcc6, 0xcca, 0xcd5, 0xce2, 0xd02, 0xd3e, 0xd46,+ 0xd4a, 0xd57, 0xd62, 0xd82, 0xdca, 0xdcf, 0xdd6, 0xdd8, 0xdf2, 0xe31, 0xe34, 0xe47, 0xeb1,+ 0xeb4, 0xebb, 0xec8, 0xf18, 0xf35, 0xf37, 0xf39, 0xf3e, 0xf71, 0xf86, 0xf8d, 0xf99, 0xfc6,+ 0x102b, 0x1056, 0x105e, 0x1062, 0x1067, 0x1071, 0x1082, 0x108f, 0x109a, 0x135d, 0x1712,+ 0x1732, 0x1752, 0x1772, 0x17b4, 0x17dd, 0x180b, 0x18a9, 0x1920, 0x1930, 0x19b0, 0x19c8,+ 0x1a17, 0x1a55, 0x1a60, 0x1a7f, 0x1b00, 0x1b34, 0x1b6b, 0x1b80, 0x1ba1, 0x1be6, 0x1c24,+ 0x1cd0, 0x1cd4, 0x1ced, 0x1cf2, 0x1dc0, 0x1dfc, 0x200c, 0x20d0, 0x2cef, 0x2d7f, 0x2de0,+ 0x302a, 0x3099, 0xa66f, 0xa674, 0xa69f, 0xa6f0, 0xa802, 0xa806, 0xa80b, 0xa823, 0xa880,+ 0xa8b4, 0xa8e0, 0xa926, 0xa947, 0xa980, 0xa9b3, 0xaa29, 0xaa43, 0xaa4c, 0xaa7b, 0xaab0,+ 0xaab2, 0xaab7, 0xaabe, 0xaac1, 0xaaeb, 0xaaf5, 0xabe3, 0xabec, 0xfb1e, 0xfe00, 0xfe20,+ 0xff9e, 0x101fd, 0x10a01, 0x10a05, 0x10a0C, 0x10a38, 0x10a3F, 0x11000, 0x11001, 0x11038,+ 0x11080, 0x11082, 0x110b0, 0x110b3, 0x110b7, 0x110b9, 0x11100, 0x11127, 0x1112c, 0x11180,+ 0x11182, 0x111b3, 0x111b6, 0x111bF, 0x116ab, 0x116ac, 0x116b0, 0x116b6, 0x16f51, 0x16f8f,+ 0x1d165, 0x1d167, 0x1d16d, 0x1d17b, 0x1d185, 0x1d1aa, 0x1d242, 0xe0100+ };+StringBuilder builder = new StringBuilder();int numChars = TestUtil.nextInt(random(), 100 * 1024, 1024 * 1024);- for (int i = 0 ; i < numChars ; ) {- builder.append(WordBreak_ExtendNumLet_chars[random().nextInt(WordBreak_ExtendNumLet_chars.length)]);+ for (int i = 0; i < numChars; ) {+ builder.append(+ WordBreak_ExtendNumLet_chars[random().nextInt(WordBreak_ExtendNumLet_chars.length)]);++i;if (random().nextBoolean()) {int numFormatExtendChars = TestUtil.nextInt(random(), 1, 8);@@ -94,7 +98,7 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {StandardTokenizer ts = new StandardTokenizer();ts.setReader(new StringReader(builder.toString()));ts.reset();- while (ts.incrementToken()) { }+ while (ts.incrementToken()) {}ts.end();ts.close();@@ -102,11 +106,11 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {ts.setMaxTokenLength(newBufferSize); // try a different buffer sizets.setReader(new StringReader(builder.toString()));ts.reset();- while (ts.incrementToken()) { }+ while (ts.incrementToken()) {}ts.end();ts.close();}-+public void testHugeDoc() throws IOException {StringBuilder sb = new StringBuilder();char whitespace[] = new char[4094];@@ -116,23 +120,24 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {String input = sb.toString();StandardTokenizer tokenizer = new StandardTokenizer();tokenizer.setReader(new StringReader(input));- BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });+ BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] {"testing", "1234"});}private Analyzer a;-+@Overridepublic void setUp() throws Exception {super.setUp();- a = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory());- return new TokenStreamComponents(tokenizer);- }- };+ a =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory());+ return new TokenStreamComponents(tokenizer);+ }+ };}-+@Overridepublic void tearDown() throws Exception {a.close();@@ -140,328 +145,528 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {}public void testArmenian() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",- new String[] { "Վիքիպեդիայի", "13", "միլիոն", "հոդվածները", "4,600", "հայերեն", "վիքիպեդիայում", "գրվել", "են", "կամավորների", "կողմից",- "ու", "համարյա", "բոլոր", "հոդվածները", "կարող", "է", "խմբագրել", "ցանկաց", "մարդ", "ով", "կարող", "է", "բացել", "Վիքիպեդիայի", "կայքը" } );+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",+ new String[] {+ "Վիքիպեդիայի",+ "13",+ "միլիոն",+ "հոդվածները",+ "4,600",+ "հայերեն",+ "վիքիպեդիայում",+ "գրվել",+ "են",+ "կամավորների",+ "կողմից",+ "ու",+ "համարյա",+ "բոլոր",+ "հոդվածները",+ "կարող",+ "է",+ "խմբագրել",+ "ցանկաց",+ "մարդ",+ "ով",+ "կարող",+ "է",+ "բացել",+ "Վիքիպեդիայի",+ "կայքը"+ });}-+public void testAmharic() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",- new String[] { "ዊኪፔድያ", "የባለ", "ብዙ", "ቋንቋ", "የተሟላ", "ትክክለኛና", "ነጻ", "መዝገበ", "ዕውቀት", "ኢንሳይክሎፒዲያ", "ነው", "ማንኛውም" } );+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",+ new String[] {+ "ዊኪፔድያ",+ "የባለ",+ "ብዙ",+ "ቋንቋ",+ "የተሟላ",+ "ትክክለኛና",+ "ነጻ",+ "መዝገበ",+ "ዕውቀት",+ "ኢንሳይክሎፒዲያ",+ "ነው",+ "ማንኛውም"+ });}-+public void testArabic() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",- new String[] { "الفيلم", "الوثائقي", "الأول", "عن", "ويكيبيديا", "يسمى", "الحقيقة", "بالأرقام", "قصة", "ويكيبيديا",- "بالإنجليزية", "Truth", "in", "Numbers", "The", "Wikipedia", "Story", "سيتم", "إطلاقه", "في", "2008" } );+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",+ new String[] {+ "الفيلم",+ "الوثائقي",+ "الأول",+ "عن",+ "ويكيبيديا",+ "يسمى",+ "الحقيقة",+ "بالأرقام",+ "قصة",+ "ويكيبيديا",+ "بالإنجليزية",+ "Truth",+ "in",+ "Numbers",+ "The",+ "Wikipedia",+ "Story",+ "سيتم",+ "إطلاقه",+ "في",+ "2008"+ });}-+public void testAramaic() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",- new String[] { "ܘܝܩܝܦܕܝܐ", "ܐܢܓܠܝܐ", "Wikipedia", "ܗܘ", "ܐܝܢܣܩܠܘܦܕܝܐ", "ܚܐܪܬܐ", "ܕܐܢܛܪܢܛ", "ܒܠܫܢ̈ܐ", "ܣܓܝܐ̈ܐ", "ܫܡܗ",- "ܐܬܐ", "ܡܢ", "ܡ̈ܠܬܐ", "ܕ", "ܘܝܩܝ", "ܘ", "ܐܝܢܣܩܠܘܦܕܝܐ"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",+ new String[] {+ "ܘܝܩܝܦܕܝܐ",+ "ܐܢܓܠܝܐ",+ "Wikipedia",+ "ܗܘ",+ "ܐܝܢܣܩܠܘܦܕܝܐ",+ "ܚܐܪܬܐ",+ "ܕܐܢܛܪܢܛ",+ "ܒܠܫܢ̈ܐ",+ "ܣܓܝܐ̈ܐ",+ "ܫܡܗ",+ "ܐܬܐ",+ "ܡܢ",+ "ܡ̈ܠܬܐ",+ "ܕ",+ "ܘܝܩܝ",+ "ܘ",+ "ܐܝܢܣܩܠܘܦܕܝܐ"+ });}-+public void testBengali() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",- new String[] { "এই", "বিশ্বকোষ", "পরিচালনা", "করে", "উইকিমিডিয়া", "ফাউন্ডেশন", "একটি", "অলাভজনক", "সংস্থা", "উইকিপিডিয়ার",- "শুরু", "১৫", "জানুয়ারি", "২০০১", "সালে", "এখন", "পর্যন্ত", "২০০টিরও", "বেশী", "ভাষায়", "উইকিপিডিয়া", "রয়েছে" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",+ new String[] {+ "এই",+ "বিশ্বকোষ",+ "পরিচালনা",+ "করে",+ "উইকিমিডিয়া",+ "ফাউন্ডেশন",+ "একটি",+ "অলাভজনক",+ "সংস্থা",+ "উইকিপিডিয়ার",+ "শুরু",+ "১৫",+ "জানুয়ারি",+ "২০০১",+ "সালে",+ "এখন",+ "পর্যন্ত",+ "২০০টিরও",+ "বেশী",+ "ভাষায়",+ "উইকিপিডিয়া",+ "রয়েছে"+ });}-+public void testFarsi() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",- new String[] { "ویکی", "پدیای", "انگلیسی", "در", "تاریخ", "۲۵", "دی", "۱۳۷۹", "به", "صورت", "مکملی",- "برای", "دانشنامهٔ", "تخصصی", "نوپدیا", "نوشته", "شد" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",+ new String[] {+ "ویکی",+ "پدیای",+ "انگلیسی",+ "در",+ "تاریخ",+ "۲۵",+ "دی",+ "۱۳۷۹",+ "به",+ "صورت",+ "مکملی",+ "برای",+ "دانشنامهٔ",+ "تخصصی",+ "نوپدیا",+ "نوشته",+ "شد"+ });}-+public void testGreek() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",- new String[] { "Γράφεται", "σε", "συνεργασία", "από", "εθελοντές", "με", "το", "λογισμικό", "wiki", "κάτι", "που",- "σημαίνει", "ότι", "άρθρα", "μπορεί", "να", "προστεθούν", "ή", "να", "αλλάξουν", "από", "τον", "καθένα" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",+ new String[] {+ "Γράφεται",+ "σε",+ "συνεργασία",+ "από",+ "εθελοντές",+ "με",+ "το",+ "λογισμικό",+ "wiki",+ "κάτι",+ "που",+ "σημαίνει",+ "ότι",+ "άρθρα",+ "μπορεί",+ "να",+ "προστεθούν",+ "ή",+ "να",+ "αλλάξουν",+ "από",+ "τον",+ "καθένα"+ });}public void testThai() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",- new String[] { "การที่ได้ต้องแสดงว่างานดี", "แล้วเธอจะไปไหน", "๑๒๓๔" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",+ new String[] {"การที่ได้ต้องแสดงว่างานดี", "แล้วเธอจะไปไหน", "๑๒๓๔"});}-+public void testLao() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ",- new String[] { "ສາທາລະນະລັດ", "ປະຊາທິປະໄຕ", "ປະຊາຊົນລາວ" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ",+ new String[] {"ສາທາລະນະລັດ", "ປະຊາທິປະໄຕ", "ປະຊາຊົນລາວ"});}-+public void testTibetan() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",- new String[] { "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག",- "མི", "ཉམས", "གོང", "འཕེལ", "དུ", "གཏོང", "བར",- "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",+ new String[] {+ "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག",+ "མི", "ཉམས", "གོང", "འཕེལ", "དུ", "གཏོང", "བར",+ "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ"+ });}-+/** For chinese, tokenize as char (these can later form bigrams or whatever)*/public void testChinese() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "我是中国人。 1234 Tests ",- new String[] { "我", "是", "中", "国", "人", "1234", "Tests"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "我是中国人。 1234 Tests ", new String[] {"我", "是", "中", "国", "人", "1234", "Tests"});}-+public void testEmpty() throws Exception {BaseTokenStreamTestCase.assertAnalyzesTo(a, "", new String[] {});BaseTokenStreamTestCase.assertAnalyzesTo(a, ".", new String[] {});BaseTokenStreamTestCase.assertAnalyzesTo(a, " ", new String[] {});}-+/* test various jira issues this analyzer is related to */-+public void testLUCENE1545() throws Exception {/** Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTRE E.* The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.* Expected result is only on token "moͤchte".*/- BaseTokenStreamTestCase.assertAnalyzesTo(a, "moͤchte", new String[] { "moͤchte" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "moͤchte", new String[] {"moͤchte"});}-+/* Tests from StandardAnalyzer, just to show behavior is similar */public void testAlphanumericSA() throws Exception {// alphanumeric tokens- BaseTokenStreamTestCase.assertAnalyzesTo(a, "B2B", new String[]{"B2B"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "2B", new String[]{"2B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "B2B", new String[] {"B2B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "2B", new String[] {"2B"});}public void testDelimitersSA() throws Exception {// other delimiters: "-", "/", ","- BaseTokenStreamTestCase.assertAnalyzesTo(a, "some-dashed-phrase", new String[]{"some", "dashed", "phrase"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "dogs,chase,cats", new String[]{"dogs", "chase", "cats"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "ac/dc", new String[]{"ac", "dc"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "some-dashed-phrase", new String[] {"some", "dashed", "phrase"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "dogs,chase,cats", new String[] {"dogs", "chase", "cats"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "ac/dc", new String[] {"ac", "dc"});}public void testApostrophesSA() throws Exception {// internal apostrophes: O'Reilly, you're, O'Reilly's- BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly", new String[]{"O'Reilly"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "you're", new String[]{"you're"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "she's", new String[]{"she's"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "Jim's", new String[]{"Jim's"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "don't", new String[]{"don't"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly's", new String[]{"O'Reilly's"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly", new String[] {"O'Reilly"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "you're", new String[] {"you're"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "she's", new String[] {"she's"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "Jim's", new String[] {"Jim's"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "don't", new String[] {"don't"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly's", new String[] {"O'Reilly's"});}public void testNumericSA() throws Exception {// floating point, serial, model numbers, ip addresses, etc.- BaseTokenStreamTestCase.assertAnalyzesTo(a, "21.35", new String[]{"21.35"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "R2D2 C3PO", new String[]{"R2D2", "C3PO"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "21.35", new String[] {"21.35"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "R2D2 C3PO", new String[] {"R2D2", "C3PO"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "216.239.63.104", new String[] {"216.239.63.104"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "216.239.63.104", new String[] {"216.239.63.104"});}public void testTextWithNumbersSA() throws Exception {// numbers- BaseTokenStreamTestCase.assertAnalyzesTo(a, "David has 5000 bones", new String[]{"David", "has", "5000", "bones"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "David has 5000 bones", new String[] {"David", "has", "5000", "bones"});}public void testVariousTextSA() throws Exception {// various- BaseTokenStreamTestCase.assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"C", "embedded", "developers", "wanted"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "FOO", "BAR"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo bar . FOO <> BAR", new String[]{"foo", "bar", "FOO", "BAR"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"QUOTED", "word"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "C embedded developers wanted", new String[] {"C", "embedded", "developers", "wanted"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "foo bar FOO BAR", new String[] {"foo", "bar", "FOO", "BAR"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "foo bar . FOO <> BAR", new String[] {"foo", "bar", "FOO", "BAR"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "\"QUOTED\" word", new String[] {"QUOTED", "word"});}public void testKoreanSA() throws Exception {// Korean words- BaseTokenStreamTestCase.assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[]{"안녕하세요", "한글입니다"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[] {"안녕하세요", "한글입니다"});}-+public void testOffsets() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "David has 5000 bones",+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "David has 5000 bones",new String[] {"David", "has", "5000", "bones"},new int[] {0, 6, 10, 15},new int[] {5, 9, 14, 20});}-+public void testTypes() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "David has 5000 bones",+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "David has 5000 bones",new String[] {"David", "has", "5000", "bones"},- new String[] { "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>" });+ new String[] {"<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>"});}-+public void testUnicodeWordBreaks() throws Exception {WordBreakTestUnicode_9_0_0 wordBreakTest = new WordBreakTestUnicode_9_0_0();wordBreakTest.test(a);}-+public void testSupplementary() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "𩬅艱鍟䇹愯瀛",+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "𩬅艱鍟䇹愯瀛",new String[] {"𩬅", "艱", "鍟", "䇹", "愯", "瀛"},- new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>" });+ new String[] {+ "<IDEOGRAPHIC>",+ "<IDEOGRAPHIC>",+ "<IDEOGRAPHIC>",+ "<IDEOGRAPHIC>",+ "<IDEOGRAPHIC>",+ "<IDEOGRAPHIC>"+ });}-+public void testKorean() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "훈민정음",- new String[] { "훈민정음" },- new String[] { "<HANGUL>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "훈민정음", new String[] {"훈민정음"}, new String[] {"<HANGUL>"});}-+public void testJapanese() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "仮名遣い カタカナ",- new String[] { "仮", "名", "遣", "い", "カタカナ" },- new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "仮名遣い カタカナ",+ new String[] {"仮", "名", "遣", "い", "カタカナ"},+ new String[] {+ "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>"+ });}-+public void testCombiningMarks() throws Exception {checkOneTerm(a, "ざ", "ざ"); // hiraganacheckOneTerm(a, "ザ", "ザ"); // katakanacheckOneTerm(a, "壹゙", "壹゙"); // ideographic- checkOneTerm(a, "아゙", "아゙"); // hangul+ checkOneTerm(a, "아゙", "아゙"); // hangul}/**- * Multiple consecutive chars in \p{WB:MidLetter}, \p{WB:MidNumLet},- * and/or \p{MidNum} should trigger a token split.+ * Multiple consecutive chars in \p{WB:MidLetter}, \p{WB:MidNumLet}, and/or \p{MidNum} should+ * trigger a token split.*/public void testMid() throws Exception {- // ':' is in \p{WB:MidLetter}, which should trigger a split unless there is a Letter char on both sides- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B", new String[] { "A:B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A::B", new String[] { "A", "B" });-- // '.' is in \p{WB:MidNumLet}, which should trigger a split unless there is a Letter or Numeric char on both sides- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2", new String[] { "1.2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B", new String[] { "A.B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1..2", new String[] { "1", "2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A..B", new String[] { "A", "B" });-- // ',' is in \p{WB:MidNum}, which should trigger a split unless there is a Numeric char on both sides- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2", new String[] { "1,2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,,2", new String[] { "1", "2" });+ // ':' is in \p{WB:MidLetter}, which should trigger a split unless there is a Letter char on+ // both sides+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B", new String[] {"A:B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A::B", new String[] {"A", "B"});++ // '.' is in \p{WB:MidNumLet}, which should trigger a split unless there is a Letter or Numeric+ // char on both sides+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2", new String[] {"1.2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B", new String[] {"A.B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1..2", new String[] {"1", "2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A..B", new String[] {"A", "B"});++ // ',' is in \p{WB:MidNum}, which should trigger a split unless there is a Numeric char on both+ // sides+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2", new String[] {"1,2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,,2", new String[] {"1", "2"});// Mixed consecutive \p{WB:MidLetter} and \p{WB:MidNumLet} should trigger a split- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.:B", new String[] { "A", "B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:.B", new String[] { "A", "B" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.:B", new String[] {"A", "B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:.B", new String[] {"A", "B"});// Mixed consecutive \p{WB:MidNum} and \p{WB:MidNumLet} should trigger a split- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,.2", new String[] { "1", "2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.,2", new String[] { "1", "2" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,.2", new String[] {"1", "2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.,2", new String[] {"1", "2"});// '_' is in \p{WB:ExtendNumLet}- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A:B", new String[] { "A:B_A:B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A::B", new String[] { "A:B_A", "B" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A:B", new String[] {"A:B_A:B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A::B", new String[] {"A:B_A", "B"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1.2", new String[] { "1.2_1.2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A.B", new String[] { "A.B_A.B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1..2", new String[] { "1.2_1", "2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A..B", new String[] { "A.B_A", "B" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1.2", new String[] {"1.2_1.2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A.B", new String[] {"A.B_A.B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1..2", new String[] {"1.2_1", "2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A..B", new String[] {"A.B_A", "B"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,2", new String[] { "1,2_1,2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,,2", new String[] { "1,2_1", "2" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,2", new String[] {"1,2_1,2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,,2", new String[] {"1,2_1", "2"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A.:B", new String[] { "C_A", "B" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A:.B", new String[] { "C_A", "B" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A.:B", new String[] {"C_A", "B"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A:.B", new String[] {"C_A", "B"});- BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1,.2", new String[] { "3_1", "2" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1.,2", new String[] { "3_1", "2" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1,.2", new String[] {"3_1", "2"});+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1.,2", new String[] {"3_1", "2"});}/** simple emoji */public void testEmoji() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "💩 💩💩",- new String[] { "💩", "💩", "💩" },- new String[] { "<EMOJI>", "<EMOJI>", "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "💩 💩💩",+ new String[] {"💩", "💩", "💩"},+ new String[] {"<EMOJI>", "<EMOJI>", "<EMOJI>"});}/** emoji zwj sequence */public void testEmojiSequence() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "👩❤️👩",- new String[] { "👩❤️👩" },- new String[] { "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "👩❤️👩", new String[] {"👩❤️👩"}, new String[] {"<EMOJI>"});}/** emoji zwj sequence with fitzpatrick modifier */public void testEmojiSequenceWithModifier() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "👨🏼⚕️",- new String[] { "👨🏼⚕️" },- new String[] { "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "👨🏼⚕️", new String[] {"👨🏼⚕️"}, new String[] {"<EMOJI>"});}/** regional indicator */public void testEmojiRegionalIndicator() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "🇺🇸🇺🇸",- new String[] { "🇺🇸", "🇺🇸" },- new String[] { "<EMOJI>", "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "🇺🇸🇺🇸", new String[] {"🇺🇸", "🇺🇸"}, new String[] {"<EMOJI>", "<EMOJI>"});}/** variation sequence */public void testEmojiVariationSequence() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "#️⃣",- new String[] { "#️⃣" },- new String[] { "<EMOJI>" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "3️⃣",- new String[] { "3️⃣",},- new String[] { "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "#️⃣", new String[] {"#️⃣"}, new String[] {"<EMOJI>"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "3️⃣",+ new String[] {+ "3️⃣",+ },+ new String[] {"<EMOJI>"});// text presentation sequences- BaseTokenStreamTestCase.assertAnalyzesTo(a, "#\uFE0E",- new String[] { },- new String[] { });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "3\uFE0E", // \uFE0E is included in \p{WB:Extend}- new String[] { "3\uFE0E",},- new String[] { "<NUM>" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "\u2B55\uFE0E", // \u2B55 = HEAVY BLACK CIRCLE- new String[] { "\u2B55",},- new String[] { "<EMOJI>" });- BaseTokenStreamTestCase.assertAnalyzesTo(a, "\u2B55\uFE0E\u200D\u2B55\uFE0E",- new String[] { "\u2B55", "\u200D\u2B55"},- new String[] { "<EMOJI>", "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(a, "#\uFE0E", new String[] {}, new String[] {});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "3\uFE0E", // \uFE0E is included in \p{WB:Extend}+ new String[] {+ "3\uFE0E",+ },+ new String[] {"<NUM>"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "\u2B55\uFE0E", // \u2B55 = HEAVY BLACK CIRCLE+ new String[] {+ "\u2B55",+ },+ new String[] {"<EMOJI>"});+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "\u2B55\uFE0E\u200D\u2B55\uFE0E",+ new String[] {"\u2B55", "\u200D\u2B55"},+ new String[] {"<EMOJI>", "<EMOJI>"});}public void testEmojiTagSequence() throws Exception {- BaseTokenStreamTestCase.assertAnalyzesTo(a, "🏴",- new String[] { "🏴" },- new String[] { "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a, "🏴", new String[] {"🏴"}, new String[] {"<EMOJI>"});}public void testEmojiTokenization() throws Exception {// simple emoji around latin- BaseTokenStreamTestCase.assertAnalyzesTo(a, "poo💩poo",- new String[] { "poo", "💩", "poo" },- new String[] { "<ALPHANUM>", "<EMOJI>", "<ALPHANUM>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "poo💩poo",+ new String[] {"poo", "💩", "poo"},+ new String[] {"<ALPHANUM>", "<EMOJI>", "<ALPHANUM>"});// simple emoji around non-latin- BaseTokenStreamTestCase.assertAnalyzesTo(a, "💩中國💩",- new String[] { "💩", "中", "國", "💩" },- new String[] { "<EMOJI>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<EMOJI>" });+ BaseTokenStreamTestCase.assertAnalyzesTo(+ a,+ "💩中國💩",+ new String[] {"💩", "中", "國", "💩"},+ new String[] {"<EMOJI>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<EMOJI>"});}-+public void testUnicodeEmojiTests() throws Exception {EmojiTokenizationTestUnicode_11_0 emojiTest = new EmojiTokenizationTestUnicode_11_0();emojiTest.test(a);}-+/** blast some random strings through the analyzer */public void testRandomStrings() throws Exception {Analyzer analyzer = new StandardAnalyzer();checkRandomData(random(), analyzer, 200 * RANDOM_MULTIPLIER);analyzer.close();}-+/** blast some random large strings through the analyzer */public void testRandomHugeStrings() throws Exception {Analyzer analyzer = new StandardAnalyzer();- checkRandomData(random(), analyzer, 20*RANDOM_MULTIPLIER, 8192);+ checkRandomData(random(), analyzer, 20 * RANDOM_MULTIPLIER, 8192);analyzer.close();}// Adds random graph after:public void testRandomHugeStringsGraphAfter() throws Exception {Random random = random();- Analyzer analyzer = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory());- TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer);- return new TokenStreamComponents(tokenizer, tokenStream);- }- };- checkRandomData(random, analyzer, 20*RANDOM_MULTIPLIER, 8192);+ Analyzer analyzer =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new StandardTokenizer(newAttributeFactory());+ TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer);+ return new TokenStreamComponents(tokenizer, tokenStream);+ }+ };+ checkRandomData(random, analyzer, 20 * RANDOM_MULTIPLIER, 8192);analyzer.close();}@@ -475,7 +680,7 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {StringBuilder bToken = new StringBuilder();// exact max length:- for(int i=0;i<StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;i++) {+ for (int i = 0; i < StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH; i++) {bToken.append('b');}@@ -489,13 +694,13 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {public void testMaxTokenLengthNonDefault() throws Exception {StandardAnalyzer a = new StandardAnalyzer();a.setMaxTokenLength(5);- assertAnalyzesTo(a, "ab cd toolong xy z", new String[]{"ab", "cd", "toolo", "ng", "xy", "z"});+ assertAnalyzesTo(a, "ab cd toolong xy z", new String[] {"ab", "cd", "toolo", "ng", "xy", "z"});a.close();}public void testSplitSurrogatePairWithSpoonFeedReader() throws Exception {String text = "12345678\ud800\udf00"; // U+D800 U+DF00 = U+10300 = 𐌀 (OLD ITALIC LETTER A)-+// Collect tokens with normal readerStandardAnalyzer a = new StandardAnalyzer();TokenStream ts = a.tokenStream("dummy", text);@@ -509,12 +714,13 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {ts.close();// Tokens from a spoon-feed reader should be the same as from a normal reader- // The 9th char is a high surrogate, so the 9-max-chars spoon-feed reader will split the surrogate pair at a read boundary+ // The 9th char is a high surrogate, so the 9-max-chars spoon-feed reader will split the+ // surrogate pair at a read boundaryReader reader = new SpoonFeedMaxCharsReaderWrapper(9, new StringReader(text));ts = a.tokenStream("dummy", reader);termAtt = ts.addAttribute(CharTermAttribute.class);ts.reset();- for (int tokenNum = 0 ; ts.incrementToken() ; ++tokenNum) {+ for (int tokenNum = 0; ts.incrementToken(); ++tokenNum) {assertEquals("token #" + tokenNum + " mismatch: ", termAtt.toString(), tokens.get(tokenNum));}ts.end();@@ -524,7 +730,7 @@ public class TestStandardAnalyzer extends BaseTokenStreamTestCase {class SpoonFeedMaxCharsReaderWrapper extends Reader {private final Reader in;- private final int maxChars;+ private final int maxChars;public SpoonFeedMaxCharsReaderWrapper(int maxChars, Reader in) {this.in = in;diff --git a/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java b/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.javaindex 758d503b53c..2efd04624a5 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java@@ -16,48 +16,44 @@*/package org.apache.lucene.analysis.standard;--import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;-import org.apache.lucene.analysis.Tokenizer;-import java.io.Reader;import java.io.StringReader;+import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;+import org.apache.lucene.analysis.Tokenizer;-/**- * Simple tests to ensure the standard lucene factories are working.- */+/** Simple tests to ensure the standard lucene factories are working. */public class TestStandardFactories extends BaseTokenStreamFactoryTestCase {- /**- * Test StandardTokenizerFactory- */+ /** Test StandardTokenizerFactory */public void testStandardTokenizer() throws Exception {Reader reader = new StringReader("Wha\u0301t's this thing do?");Tokenizer stream = tokenizerFactory("Standard").create(newAttributeFactory());stream.setReader(reader);- assertTokenStreamContents(stream,- new String[]{"Wha\u0301t's", "this", "thing", "do"});+ assertTokenStreamContents(stream, new String[] {"Wha\u0301t's", "this", "thing", "do"});}-+public void testStandardTokenizerMaxTokenLength() throws Exception {StringBuilder builder = new StringBuilder();- for (int i = 0 ; i < 100 ; ++i) {+ for (int i = 0; i < 100; ++i) {builder.append("abcdefg"); // 7 * 100 = 700 char "word"}String longWord = builder.toString();String content = "one two three " + longWord + " four five six";Reader reader = new StringReader(content);- Tokenizer stream = tokenizerFactory("Standard",- "maxTokenLength", "1000").create(newAttributeFactory());+ Tokenizer stream =+ tokenizerFactory("Standard", "maxTokenLength", "1000").create(newAttributeFactory());stream.setReader(reader);- assertTokenStreamContents(stream,- new String[]{"one", "two", "three", longWord, "four", "five", "six"});+ assertTokenStreamContents(+ stream, new String[] {"one", "two", "three", longWord, "four", "five", "six"});}/** Test that bogus arguments result in exception */public void testBogusArguments() throws Exception {- IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {- tokenizerFactory("Standard", "bogusArg", "bogusValue");- });+ IllegalArgumentException expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ tokenizerFactory("Standard", "bogusArg", "bogusValue");+ });assertTrue(expected.getMessage().contains("Unknown parameters"));}}diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.javaindex 6c01d5cdd74..54f23e7189f 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java@@ -16,7 +16,6 @@*/package org.apache.lucene.analysis.tokenattributes;-import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.LuceneTestCase;diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.javaindex bbcf10db19a..53af58b8f68 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java@@ -16,17 +16,15 @@*/package org.apache.lucene.analysis.tokenattributes;--import org.apache.lucene.util.AttributeImpl;-import org.apache.lucene.util.LuceneTestCase;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.TestUtil;-import java.nio.CharBuffer;-import java.util.HashMap;import java.util.Formatter;+import java.util.HashMap;import java.util.Locale;import java.util.regex.Pattern;+import org.apache.lucene.util.AttributeImpl;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.LuceneTestCase;+import org.apache.lucene.util.TestUtil;public class TestCharTermAttributeImpl extends LuceneTestCase {@@ -34,8 +32,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {CharTermAttributeImpl t = new CharTermAttributeImpl();char[] content = "hello".toCharArray();t.copyBuffer(content, 0, content.length);- for (int i = 0; i < 2000; i++)- {+ for (int i = 0; i < 2000; i++) {t.resizeBuffer(i);assertTrue(i <= t.buffer().length);assertEquals("hello", t.toString());@@ -46,17 +43,18 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {CharTermAttributeImpl t = new CharTermAttributeImpl();char[] content = "hello".toCharArray();t.copyBuffer(content, 0, content.length);- expectThrows(IndexOutOfBoundsException.class, () -> {- t.setLength(-1);- });+ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.setLength(-1);+ });}@Slowpublic void testGrow() {CharTermAttributeImpl t = new CharTermAttributeImpl();StringBuilder buf = new StringBuilder("ab");- for (int i = 0; i < 20; i++)- {+ for (int i = 0; i < 20; i++) {char[] content = buf.toString().toCharArray();t.copyBuffer(content, 0, content.length);assertEquals(buf.length(), t.length());@@ -68,8 +66,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {// now as a StringBuilder, first variantt = new CharTermAttributeImpl();buf = new StringBuilder("ab");- for (int i = 0; i < 20; i++)- {+ for (int i = 0; i < 20; i++) {t.setEmpty().append(buf);assertEquals(buf.length(), t.length());assertEquals(buf.toString(), t.toString());@@ -80,8 +77,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {// Test for slow growth to a long termt = new CharTermAttributeImpl();buf = new StringBuilder("a");- for (int i = 0; i < 20000; i++)- {+ for (int i = 0; i < 20000; i++) {t.setEmpty().append(buf);assertEquals(buf.length(), t.length());assertEquals(buf.toString(), t.toString());@@ -109,7 +105,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertEquals(t.toString(), copy.toString());assertNotSame(buf, copy.buffer());}-+public void testEquals() throws Exception {CharTermAttributeImpl t1a = new CharTermAttributeImpl();char[] content1a = "hello".toCharArray();@@ -124,7 +120,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertFalse(t1a.equals(t2));assertFalse(t2.equals(t1b));}-+public void testCopyTo() throws Exception {CharTermAttributeImpl t = new CharTermAttributeImpl();CharTermAttributeImpl copy = assertCopyIsEqual(t);@@ -139,30 +135,34 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertEquals(t.toString(), copy.toString());assertNotSame(buf, copy.buffer());}-+public void testAttributeReflection() throws Exception {CharTermAttributeImpl t = new CharTermAttributeImpl();t.append("foobar");- TestUtil.assertAttributeReflection(t, new HashMap<String, Object>() {{- put(CharTermAttribute.class.getName() + "#term", "foobar");- put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));- }});+ TestUtil.assertAttributeReflection(+ t,+ new HashMap<String, Object>() {+ {+ put(CharTermAttribute.class.getName() + "#term", "foobar");+ put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));+ }+ });}-+public void testCharSequenceInterface() {- final String s = "0123456789";+ final String s = "0123456789";final CharTermAttributeImpl t = new CharTermAttributeImpl();t.append(s);-+assertEquals(s.length(), t.length());- assertEquals("12", t.subSequence(1,3).toString());- assertEquals(s, t.subSequence(0,s.length()).toString());-+ assertEquals("12", t.subSequence(1, 3).toString());+ assertEquals(s, t.subSequence(0, s.length()).toString());+assertTrue(Pattern.matches("01\\d+", t));- assertTrue(Pattern.matches("34", t.subSequence(3,5)));-- assertEquals(s.subSequence(3,7).toString(), t.subSequence(3,7).toString());-+ assertTrue(Pattern.matches("34", t.subSequence(3, 5)));++ assertEquals(s.subSequence(3, 7).toString(), t.subSequence(3, 7).toString());+for (int i = 0; i < s.length(); i++) {assertTrue(t.charAt(i) == s.charAt(i));}@@ -200,24 +200,34 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertEquals("4test", t.toString());t.append((CharSequence) t2, 1, 2);assertEquals("4teste", t.toString());-- expectThrows(IndexOutOfBoundsException.class, () -> {- t.append((CharSequence) t2, 1, 5);- });-- expectThrows(IndexOutOfBoundsException.class, () -> {- t.append((CharSequence) t2, 1, 0);- });-++ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.append((CharSequence) t2, 1, 5);+ });++ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.append((CharSequence) t2, 1, 0);+ });+t.append((CharSequence) null);assertEquals("4testenull", t.toString());}-+public void testAppendableInterfaceWithLongSequences() {CharTermAttributeImpl t = new CharTermAttributeImpl();t.append((CharSequence) "01234567890123456789012345678901234567890123456789");- t.append((CharSequence) CharBuffer.wrap("01234567890123456789012345678901234567890123456789".toCharArray()), 3, 50);- assertEquals("0123456789012345678901234567890123456789012345678934567890123456789012345678901234567890123456789", t.toString());+ t.append(+ (CharSequence)+ CharBuffer.wrap("01234567890123456789012345678901234567890123456789".toCharArray()),+ 3,+ 50);+ assertEquals(+ "0123456789012345678901234567890123456789012345678934567890123456789012345678901234567890123456789",+ t.toString());t.setEmpty().append((CharSequence) new StringBuilder("01234567890123456789"), 5, 17);assertEquals((CharSequence) "567890123456", t.toString());t.append(new StringBuffer(t));@@ -227,22 +237,34 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertEquals("345678901234567", buf.toString());t.setEmpty().append(buf, 1, 14);assertEquals("4567890123456", t.toString());-+// finally use a completely custom CharSequence that is not catched by instanceof checksfinal String longTestString = "012345678901234567890123456789";- t.append(new CharSequence() {- @Override- public char charAt(int i) { return longTestString.charAt(i); }- @Override- public int length() { return longTestString.length(); }- @Override- public CharSequence subSequence(int start, int end) { return longTestString.subSequence(start, end); }- @Override- public String toString() { return longTestString; }- });- assertEquals("4567890123456"+longTestString, t.toString());+ t.append(+ new CharSequence() {+ @Override+ public char charAt(int i) {+ return longTestString.charAt(i);+ }++ @Override+ public int length() {+ return longTestString.length();+ }++ @Override+ public CharSequence subSequence(int start, int end) {+ return longTestString.subSequence(start, end);+ }++ @Override+ public String toString() {+ return longTestString;+ }+ });+ assertEquals("4567890123456" + longTestString, t.toString());}-+public void testNonCharSequenceAppend() {CharTermAttributeImpl t = new CharTermAttributeImpl();t.append("0123456789");@@ -259,27 +281,35 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {t.append((CharTermAttribute) null);assertEquals("012345678901234567890123456789testnullnullnull", t.toString());}-+public void testExceptions() {CharTermAttributeImpl t = new CharTermAttributeImpl();t.append("test");assertEquals("test", t.toString());- expectThrows(IndexOutOfBoundsException.class, () -> {- t.charAt(-1);- });+ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.charAt(-1);+ });- expectThrows(IndexOutOfBoundsException.class, () -> {- t.charAt(4);- });+ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.charAt(4);+ });- expectThrows(IndexOutOfBoundsException.class, () -> {- t.subSequence(0, 5);- });+ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.subSequence(0, 5);+ });- expectThrows(IndexOutOfBoundsException.class, () -> {- t.subSequence(5, 0);- });+ expectThrows(+ IndexOutOfBoundsException.class,+ () -> {+ t.subSequence(5, 0);+ });}public static <T extends AttributeImpl> T assertCloneIsEqual(T att) {@@ -298,9 +328,9 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode());return copy;}-+/*-+// test speed of the dynamic instanceof checks in append(CharSequence),// to find the best max length for the generic while (start<end) loop:public void testAppendPerf() {@@ -334,7 +364,7 @@ public class TestCharTermAttributeImpl extends LuceneTestCase {long endTime = System.currentTimeMillis();System.out.println("Time: " + (endTime-startTime)/1000.0 + " s");}-+*/}diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.javaindex c673fc65be9..8e440962630 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestPackedTokenAttributeImpl.java@@ -16,21 +16,19 @@*/package org.apache.lucene.analysis.tokenattributes;-+import java.io.StringReader;+import java.util.HashMap;import org.apache.lucene.analysis.MockTokenizer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;-import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.TestUtil;-import java.io.StringReader;-import java.util.HashMap;-public class TestPackedTokenAttributeImpl extends LuceneTestCase {/* the CharTermAttributeStuff is tested by TestCharTermAttributeImpl */-+public void testClone() throws Exception {PackedTokenAttributeImpl t = new PackedTokenAttributeImpl();t.setOffset(0, 5);@@ -41,7 +39,7 @@ public class TestPackedTokenAttributeImpl extends LuceneTestCase {assertEquals(t.toString(), copy.toString());assertNotSame(buf, copy.buffer());}-+public void testCopyTo() throws Exception {PackedTokenAttributeImpl t = new PackedTokenAttributeImpl();PackedTokenAttributeImpl copy = TestCharTermAttributeImpl.assertCopyIsEqual(t);@@ -57,42 +55,55 @@ public class TestPackedTokenAttributeImpl extends LuceneTestCase {assertEquals(t.toString(), copy.toString());assertNotSame(buf, copy.buffer());}-+public void testPackedTokenAttributeFactory() throws Exception {- TokenStream ts = new MockTokenizer(TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);- ((Tokenizer)ts).setReader(new StringReader("foo bar"));-- assertTrue("CharTermAttribute is not implemented by Token",- ts.addAttribute(CharTermAttribute.class) instanceof PackedTokenAttributeImpl);- assertTrue("OffsetAttribute is not implemented by Token",- ts.addAttribute(OffsetAttribute.class) instanceof PackedTokenAttributeImpl);- assertTrue("PositionIncrementAttribute is not implemented by Token",- ts.addAttribute(PositionIncrementAttribute.class) instanceof PackedTokenAttributeImpl);- assertTrue("TypeAttribute is not implemented by Token",- ts.addAttribute(TypeAttribute.class) instanceof PackedTokenAttributeImpl);+ TokenStream ts =+ new MockTokenizer(+ TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY,+ MockTokenizer.WHITESPACE,+ false,+ MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);+ ((Tokenizer) ts).setReader(new StringReader("foo bar"));++ assertTrue(+ "CharTermAttribute is not implemented by Token",+ ts.addAttribute(CharTermAttribute.class) instanceof PackedTokenAttributeImpl);+ assertTrue(+ "OffsetAttribute is not implemented by Token",+ ts.addAttribute(OffsetAttribute.class) instanceof PackedTokenAttributeImpl);+ assertTrue(+ "PositionIncrementAttribute is not implemented by Token",+ ts.addAttribute(PositionIncrementAttribute.class) instanceof PackedTokenAttributeImpl);+ assertTrue(+ "TypeAttribute is not implemented by Token",+ ts.addAttribute(TypeAttribute.class) instanceof PackedTokenAttributeImpl);- assertTrue("FlagsAttribute is not implemented by FlagsAttributeImpl",- ts.addAttribute(FlagsAttribute.class) instanceof FlagsAttributeImpl);+ assertTrue(+ "FlagsAttribute is not implemented by FlagsAttributeImpl",+ ts.addAttribute(FlagsAttribute.class) instanceof FlagsAttributeImpl);}public void testAttributeReflection() throws Exception {PackedTokenAttributeImpl t = new PackedTokenAttributeImpl();t.append("foobar");- t.setOffset(6, 22);+ t.setOffset(6, 22);t.setPositionIncrement(3);t.setPositionLength(11);t.setType("foobar");t.setTermFrequency(42);- TestUtil.assertAttributeReflection(t,- new HashMap<String, Object>() {{- put(CharTermAttribute.class.getName() + "#term", "foobar");- put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));- put(OffsetAttribute.class.getName() + "#startOffset", 6);- put(OffsetAttribute.class.getName() + "#endOffset", 22);- put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 3);- put(PositionLengthAttribute.class.getName() + "#positionLength", 11);- put(TypeAttribute.class.getName() + "#type", "foobar");- put(TermFrequencyAttribute.class.getName() + "#termFrequency", 42);- }});+ TestUtil.assertAttributeReflection(+ t,+ new HashMap<String, Object>() {+ {+ put(CharTermAttribute.class.getName() + "#term", "foobar");+ put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));+ put(OffsetAttribute.class.getName() + "#startOffset", 6);+ put(OffsetAttribute.class.getName() + "#endOffset", 22);+ put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 3);+ put(PositionLengthAttribute.class.getName() + "#positionLength", 11);+ put(TypeAttribute.class.getName() + "#type", "foobar");+ put(TermFrequencyAttribute.class.getName() + "#termFrequency", 42);+ }+ });}}diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.javaindex 0f752dada38..f8832d97248 100644--- a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java+++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpl.java@@ -16,33 +16,42 @@*/package org.apache.lucene.analysis.tokenattributes;--import org.apache.lucene.util.TestUtil;-import org.apache.lucene.util.LuceneTestCase;-import java.util.Collections;import java.util.HashMap;+import org.apache.lucene.util.LuceneTestCase;+import org.apache.lucene.util.TestUtil;public class TestSimpleAttributeImpl extends LuceneTestCase {// this checks using reflection API if the defaults are correctpublic void testAttributes() {- TestUtil.assertAttributeReflection(new PositionIncrementAttributeImpl(),- Collections.singletonMap(PositionIncrementAttribute.class.getName() + "#positionIncrement", 1));- TestUtil.assertAttributeReflection(new PositionLengthAttributeImpl(),+ TestUtil.assertAttributeReflection(+ new PositionIncrementAttributeImpl(),+ Collections.singletonMap(+ PositionIncrementAttribute.class.getName() + "#positionIncrement", 1));+ TestUtil.assertAttributeReflection(+ new PositionLengthAttributeImpl(),Collections.singletonMap(PositionLengthAttribute.class.getName() + "#positionLength", 1));- TestUtil.assertAttributeReflection(new FlagsAttributeImpl(),+ TestUtil.assertAttributeReflection(+ new FlagsAttributeImpl(),Collections.singletonMap(FlagsAttribute.class.getName() + "#flags", 0));- TestUtil.assertAttributeReflection(new TypeAttributeImpl(),- Collections.singletonMap(TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE));- TestUtil.assertAttributeReflection(new PayloadAttributeImpl(),+ TestUtil.assertAttributeReflection(+ new TypeAttributeImpl(),+ Collections.singletonMap(+ TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE));+ TestUtil.assertAttributeReflection(+ new PayloadAttributeImpl(),Collections.singletonMap(PayloadAttribute.class.getName() + "#payload", null));- TestUtil.assertAttributeReflection(new KeywordAttributeImpl(),+ TestUtil.assertAttributeReflection(+ new KeywordAttributeImpl(),Collections.singletonMap(KeywordAttribute.class.getName() + "#keyword", false));- TestUtil.assertAttributeReflection(new OffsetAttributeImpl(), new HashMap<String, Object>() {{- put(OffsetAttribute.class.getName() + "#startOffset", 0);- put(OffsetAttribute.class.getName() + "#endOffset", 0);- }});+ TestUtil.assertAttributeReflection(+ new OffsetAttributeImpl(),+ new HashMap<String, Object>() {+ {+ put(OffsetAttribute.class.getName() + "#startOffset", 0);+ put(OffsetAttribute.class.getName() + "#endOffset", 0);+ }+ });}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/DefaultEncoder.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/DefaultEncoder.javaindex baae18ae0cd..90e63d96730 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/DefaultEncoder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/DefaultEncoder.java@@ -16,16 +16,12 @@*/package org.apache.lucene.search.highlight;-/**- * Simple {@link Encoder} implementation that does not modify the output- *- */+/** Simple {@link Encoder} implementation that does not modify the output */public class DefaultEncoder implements Encoder {- public DefaultEncoder() {- }+ public DefaultEncoder() {}@Overridepublic String encodeText(String originalText) {return originalText;}-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Encoder.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Encoder.javaindex eb2a95782f8..c181cc5e6fa 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Encoder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Encoder.java@@ -16,13 +16,8 @@*/package org.apache.lucene.search.highlight;-/**- * Encodes original text. The Encoder works with the {@link Formatter} to generate output.- *- */+/** Encodes original text. The Encoder works with the {@link Formatter} to generate output. */public interface Encoder {- /**- * @param originalText The section of text being output- */+ /** @param originalText The section of text being output */String encodeText(String originalText);-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.javaindex 8b010efdf83..7d9190eb97e 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Formatter.java@@ -16,16 +16,14 @@*/package org.apache.lucene.search.highlight;/**- * Processes terms found in the original text, typically by applying some form- * of mark-up to highlight terms in HTML search results pages.- *+ * Processes terms found in the original text, typically by applying some form of mark-up to+ * highlight terms in HTML search results pages.*/-public interface Formatter-{+public interface Formatter {/*** @param originalText The section of text being considered for markup- * @param tokenGroup contains one or several overlapping Tokens along with- * their scores and positions.+ * @param tokenGroup contains one or several overlapping Tokens along with their scores and+ * positions.*/String highlightTerm(String originalText, TokenGroup tokenGroup);}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.javaindex 8c0187cebf9..1e79129c297 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Fragmenter.java@@ -15,31 +15,28 @@* limitations under the License.*/package org.apache.lucene.search.highlight;+import org.apache.lucene.analysis.TokenStream;/**- * Implements the policy for breaking text into multiple fragments for- * consideration by the {@link Highlighter} class. A sophisticated- * implementation may do this on the basis of detecting end of sentences in the- * text.+ * Implements the policy for breaking text into multiple fragments for consideration by the {@link+ * Highlighter} class. A sophisticated implementation may do this on the basis of detecting end of+ * sentences in the text.*/public interface Fragmenter {/**- * Initializes the Fragmenter. You can grab references to the Attributes you are- * interested in from tokenStream and then access the values in {@link #isNewFragment()}.- *+ * Initializes the Fragmenter. You can grab references to the Attributes you are interested in+ * from tokenStream and then access the values in {@link #isNewFragment()}.+ ** @param originalText the original source text* @param tokenStream the {@link TokenStream} to be fragmented*/public void start(String originalText, TokenStream tokenStream);-/**- * Test to see if this token from the stream should be held in a new- * TextFragment. Every time this is called, the TokenStream- * passed to start(String, TokenStream) will have been incremented.- *+ * Test to see if this token from the stream should be held in a new TextFragment. Every time this+ * is called, the TokenStream passed to start(String, TokenStream) will have been incremented.*/public boolean isNewFragment();}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.javaindex 8d4250a5b7c..9178f44703f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/GradientFormatter.java@@ -15,213 +15,171 @@* limitations under the License.*/package org.apache.lucene.search.highlight;-/**- * Formats text with different color intensity depending on the score of the- * term.- *- */-public class GradientFormatter implements Formatter-{- private float maxScore;-- int fgRMin, fgGMin, fgBMin;-- int fgRMax, fgGMax, fgBMax;-- protected boolean highlightForeground;-- int bgRMin, bgGMin, bgBMin;-- int bgRMax, bgGMax, bgBMax;-- protected boolean highlightBackground;-- /**- * Sets the color range for the IDF scores- *- * @param maxScore- * The score (and above) displayed as maxColor (See {@link QueryScorer#getMaxTermWeight()}- * which can be used to calibrate scoring scale)- * @param minForegroundColor- * The hex color used for representing IDF scores of zero eg- * #FFFFFF (white) or null if no foreground color required- * @param maxForegroundColor- * The largest hex color used for representing IDF scores eg- * #000000 (black) or null if no foreground color required- * @param minBackgroundColor- * The hex color used for representing IDF scores of zero eg- * #FFFFFF (white) or null if no background color required- * @param maxBackgroundColor- * The largest hex color used for representing IDF scores eg- * #000000 (black) or null if no background color required- */- public GradientFormatter(float maxScore, String minForegroundColor,- String maxForegroundColor, String minBackgroundColor,- String maxBackgroundColor)- {- highlightForeground = (minForegroundColor != null)- && (maxForegroundColor != null);- if (highlightForeground)- {- if (minForegroundColor.length() != 7)- {- throw new IllegalArgumentException(- "minForegroundColor is not 7 bytes long eg a hex "- + "RGB value such as #FFFFFF");- }- if (maxForegroundColor.length() != 7)- {- throw new IllegalArgumentException(- "minForegroundColor is not 7 bytes long eg a hex "- + "RGB value such as #FFFFFF");- }- fgRMin = hexToInt(minForegroundColor.substring(1, 3));- fgGMin = hexToInt(minForegroundColor.substring(3, 5));- fgBMin = hexToInt(minForegroundColor.substring(5, 7));-- fgRMax = hexToInt(maxForegroundColor.substring(1, 3));- fgGMax = hexToInt(maxForegroundColor.substring(3, 5));- fgBMax = hexToInt(maxForegroundColor.substring(5, 7));- }-- highlightBackground = (minBackgroundColor != null)- && (maxBackgroundColor != null);- if (highlightBackground)- {- if (minBackgroundColor.length() != 7)- {- throw new IllegalArgumentException(- "minBackgroundColor is not 7 bytes long eg a hex "- + "RGB value such as #FFFFFF");- }- if (maxBackgroundColor.length() != 7)- {- throw new IllegalArgumentException(- "minBackgroundColor is not 7 bytes long eg a hex "- + "RGB value such as #FFFFFF");- }- bgRMin = hexToInt(minBackgroundColor.substring(1, 3));- bgGMin = hexToInt(minBackgroundColor.substring(3, 5));- bgBMin = hexToInt(minBackgroundColor.substring(5, 7));-- bgRMax = hexToInt(maxBackgroundColor.substring(1, 3));- bgGMax = hexToInt(maxBackgroundColor.substring(3, 5));- bgBMax = hexToInt(maxBackgroundColor.substring(5, 7));- }- // this.corpusReader = corpusReader;- this.maxScore = maxScore;- // totalNumDocs = corpusReader.numDocs();+/** Formats text with different color intensity depending on the score of the term. */+public class GradientFormatter implements Formatter {+ private float maxScore;++ int fgRMin, fgGMin, fgBMin;+ int fgRMax, fgGMax, fgBMax;++ protected boolean highlightForeground;++ int bgRMin, bgGMin, bgBMin;+ int bgRMax, bgGMax, bgBMax;++ protected boolean highlightBackground;++ /**+ * Sets the color range for the IDF scores+ *+ * @param maxScore The score (and above) displayed as maxColor (See {@link+ * QueryScorer#getMaxTermWeight()} which can be used to calibrate scoring scale)+ * @param minForegroundColor The hex color used for representing IDF scores of zero eg #FFFFFF+ * (white) or null if no foreground color required+ * @param maxForegroundColor The largest hex color used for representing IDF scores eg #000000+ * (black) or null if no foreground color required+ * @param minBackgroundColor The hex color used for representing IDF scores of zero eg #FFFFFF+ * (white) or null if no background color required+ * @param maxBackgroundColor The largest hex color used for representing IDF scores eg #000000+ * (black) or null if no background color required+ */+ public GradientFormatter(+ float maxScore,+ String minForegroundColor,+ String maxForegroundColor,+ String minBackgroundColor,+ String maxBackgroundColor) {+ highlightForeground = (minForegroundColor != null) && (maxForegroundColor != null);+ if (highlightForeground) {+ if (minForegroundColor.length() != 7) {+ throw new IllegalArgumentException(+ "minForegroundColor is not 7 bytes long eg a hex " + "RGB value such as #FFFFFF");+ }+ if (maxForegroundColor.length() != 7) {+ throw new IllegalArgumentException(+ "minForegroundColor is not 7 bytes long eg a hex " + "RGB value such as #FFFFFF");+ }+ fgRMin = hexToInt(minForegroundColor.substring(1, 3));+ fgGMin = hexToInt(minForegroundColor.substring(3, 5));+ fgBMin = hexToInt(minForegroundColor.substring(5, 7));++ fgRMax = hexToInt(maxForegroundColor.substring(1, 3));+ fgGMax = hexToInt(maxForegroundColor.substring(3, 5));+ fgBMax = hexToInt(maxForegroundColor.substring(5, 7));}- @Override- public String highlightTerm(String originalText, TokenGroup tokenGroup)- {- if (tokenGroup.getTotalScore() == 0)- return originalText;- float score = tokenGroup.getTotalScore();- if (score == 0)- {- return originalText;- }- StringBuilder sb = new StringBuilder();- sb.append("<font ");- if (highlightForeground)- {- sb.append("color=\"");- sb.append(getForegroundColorString(score));- sb.append("\" ");- }- if (highlightBackground)- {- sb.append("bgcolor=\"");- sb.append(getBackgroundColorString(score));- sb.append("\" ");- }- sb.append(">");- sb.append(originalText);- sb.append("</font>");- return sb.toString();+ highlightBackground = (minBackgroundColor != null) && (maxBackgroundColor != null);+ if (highlightBackground) {+ if (minBackgroundColor.length() != 7) {+ throw new IllegalArgumentException(+ "minBackgroundColor is not 7 bytes long eg a hex " + "RGB value such as #FFFFFF");+ }+ if (maxBackgroundColor.length() != 7) {+ throw new IllegalArgumentException(+ "minBackgroundColor is not 7 bytes long eg a hex " + "RGB value such as #FFFFFF");+ }+ bgRMin = hexToInt(minBackgroundColor.substring(1, 3));+ bgGMin = hexToInt(minBackgroundColor.substring(3, 5));+ bgBMin = hexToInt(minBackgroundColor.substring(5, 7));++ bgRMax = hexToInt(maxBackgroundColor.substring(1, 3));+ bgGMax = hexToInt(maxBackgroundColor.substring(3, 5));+ bgBMax = hexToInt(maxBackgroundColor.substring(5, 7));}-- protected String getForegroundColorString(float score)- {- int rVal = getColorVal(fgRMin, fgRMax, score);- int gVal = getColorVal(fgGMin, fgGMax, score);- int bVal = getColorVal(fgBMin, fgBMax, score);- StringBuilder sb = new StringBuilder();- sb.append("#");- sb.append(intToHex(rVal));- sb.append(intToHex(gVal));- sb.append(intToHex(bVal));- return sb.toString();+ // this.corpusReader = corpusReader;+ this.maxScore = maxScore;+ // totalNumDocs = corpusReader.numDocs();+ }++ @Override+ public String highlightTerm(String originalText, TokenGroup tokenGroup) {+ if (tokenGroup.getTotalScore() == 0) return originalText;+ float score = tokenGroup.getTotalScore();+ if (score == 0) {+ return originalText;}-- protected String getBackgroundColorString(float score)- {- int rVal = getColorVal(bgRMin, bgRMax, score);- int gVal = getColorVal(bgGMin, bgGMax, score);- int bVal = getColorVal(bgBMin, bgBMax, score);- StringBuilder sb = new StringBuilder();- sb.append("#");- sb.append(intToHex(rVal));- sb.append(intToHex(gVal));- sb.append(intToHex(bVal));- return sb.toString();+ StringBuilder sb = new StringBuilder();+ sb.append("<font ");+ if (highlightForeground) {+ sb.append("color=\"");+ sb.append(getForegroundColorString(score));+ sb.append("\" ");}-- private int getColorVal(int colorMin, int colorMax, float score)- {- if (colorMin == colorMax)- {- return colorMin;- }- float scale = Math.abs(colorMin - colorMax);- float relScorePercent = Math.min(maxScore, score) / maxScore;- float colScore = scale * relScorePercent;- return Math.min(colorMin, colorMax) + (int) colScore;+ if (highlightBackground) {+ sb.append("bgcolor=\"");+ sb.append(getBackgroundColorString(score));+ sb.append("\" ");}-- private static char hexDigits[] = { '0', '1', '2', '3', '4', '5', '6', '7',- '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };-- private static String intToHex(int i)- {- return "" + hexDigits[(i & 0xF0) >> 4] + hexDigits[i & 0x0F];+ sb.append(">");+ sb.append(originalText);+ sb.append("</font>");+ return sb.toString();+ }++ protected String getForegroundColorString(float score) {+ int rVal = getColorVal(fgRMin, fgRMax, score);+ int gVal = getColorVal(fgGMin, fgGMax, score);+ int bVal = getColorVal(fgBMin, fgBMax, score);+ StringBuilder sb = new StringBuilder();+ sb.append("#");+ sb.append(intToHex(rVal));+ sb.append(intToHex(gVal));+ sb.append(intToHex(bVal));+ return sb.toString();+ }++ protected String getBackgroundColorString(float score) {+ int rVal = getColorVal(bgRMin, bgRMax, score);+ int gVal = getColorVal(bgGMin, bgGMax, score);+ int bVal = getColorVal(bgBMin, bgBMax, score);+ StringBuilder sb = new StringBuilder();+ sb.append("#");+ sb.append(intToHex(rVal));+ sb.append(intToHex(gVal));+ sb.append(intToHex(bVal));+ return sb.toString();+ }++ private int getColorVal(int colorMin, int colorMax, float score) {+ if (colorMin == colorMax) {+ return colorMin;}-- /**- * Converts a hex string into an int. Integer.parseInt(hex, 16) assumes the- * input is nonnegative unless there is a preceding minus sign. This method- * reads the input as twos complement instead, so if the input is 8 bytes- * long, it will correctly restore a negative int produced by- * Integer.toHexString() but not necessarily one produced by- * Integer.toString(x,16) since that method will produce a string like '-FF'- * for negative integer values.- *- * @param hex- * A string in capital or lower case hex, of no more then 16- * characters.- * @throws NumberFormatException- * if the string is more than 16 characters long, or if any- * character is not in the set [0-9a-fA-f]- */- public static final int hexToInt(String hex)- {- int len = hex.length();- if (len > 16)- throw new NumberFormatException();-- int l = 0;- for (int i = 0; i < len; i++)- {- l <<= 4;- int c = Character.digit(hex.charAt(i), 16);- if (c < 0)- throw new NumberFormatException();- l |= c;- }- return l;+ float scale = Math.abs(colorMin - colorMax);+ float relScorePercent = Math.min(maxScore, score) / maxScore;+ float colScore = scale * relScorePercent;+ return Math.min(colorMin, colorMax) + (int) colScore;+ }++ private static char hexDigits[] = {+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'+ };++ private static String intToHex(int i) {+ return "" + hexDigits[(i & 0xF0) >> 4] + hexDigits[i & 0x0F];+ }++ /**+ * Converts a hex string into an int. Integer.parseInt(hex, 16) assumes the input is nonnegative+ * unless there is a preceding minus sign. This method reads the input as twos complement instead,+ * so if the input is 8 bytes long, it will correctly restore a negative int produced by+ * Integer.toHexString() but not necessarily one produced by Integer.toString(x,16) since that+ * method will produce a string like '-FF' for negative integer values.+ *+ * @param hex A string in capital or lower case hex, of no more then 16 characters.+ * @throws NumberFormatException if the string is more than 16 characters long, or if any+ * character is not in the set [0-9a-fA-f]+ */+ public static final int hexToInt(String hex) {+ int len = hex.length();+ if (len > 16) throw new NumberFormatException();++ int l = 0;+ for (int i = 0; i < len; i++) {+ l <<= 4;+ int c = Character.digit(hex.charAt(i), 16);+ if (c < 0) throw new NumberFormatException();+ l |= c;}-+ return l;+ }}-diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.javaindex 6f3ea78a0c2..b0dee91da65 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java@@ -20,7 +20,6 @@ import java.io.IOException;import java.util.ArrayList;import java.util.Iterator;import java.util.Objects;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;@@ -28,15 +27,13 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.util.PriorityQueue;/**- * Marks up highlighted terms found in the best sections of- * text, using configurable {@link Fragmenter}, {@link Scorer}, {@link Formatter},- * {@link Encoder} and tokenizers.+ * Marks up highlighted terms found in the best sections of text, using configurable {@link+ * Fragmenter}, {@link Scorer}, {@link Formatter}, {@link Encoder} and tokenizers.*- * This is Lucene's original Highlighter; there are others.+ * <p>This is Lucene's original Highlighter; there are others.*/-public class Highlighter-{- public static final int DEFAULT_MAX_CHARS_TO_ANALYZE = 50*1024;+public class Highlighter {+ public static final int DEFAULT_MAX_CHARS_TO_ANALYZE = 50 * 1024;private Formatter formatter;private Encoder encoder;@@ -44,18 +41,15 @@ public class Highlighterprivate int maxDocCharsToAnalyze = DEFAULT_MAX_CHARS_TO_ANALYZE;private Fragmenter textFragmenter = new SimpleFragmenter();- public Highlighter(Scorer fragmentScorer)- {- this(new SimpleHTMLFormatter(),fragmentScorer);+ public Highlighter(Scorer fragmentScorer) {+ this(new SimpleHTMLFormatter(), fragmentScorer);}- public Highlighter(Formatter formatter, Scorer fragmentScorer)- {- this(formatter,new DefaultEncoder(),fragmentScorer);+ public Highlighter(Formatter formatter, Scorer fragmentScorer) {+ this(formatter, new DefaultEncoder(), fragmentScorer);}- public Highlighter(Formatter formatter, Encoder encoder, Scorer fragmentScorer)- {+ public Highlighter(Formatter formatter, Encoder encoder, Scorer fragmentScorer) {ensureArgumentNotNull(formatter, "'formatter' must not be null");ensureArgumentNotNull(encoder, "'encoder' must not be null");ensureArgumentNotNull(fragmentScorer, "'fragmentScorer' must not be null");@@ -66,140 +60,118 @@ public class Highlighter}/**- * Highlights chosen terms in a text, extracting the most relevant section.- * This is a convenience method that calls- * {@link #getBestFragment(TokenStream, String)}+ * Highlights chosen terms in a text, extracting the most relevant section. This is a convenience+ * method that calls {@link #getBestFragment(TokenStream, String)}*- * @param analyzer the analyzer that will be used to split <code>text</code>- * into chunks+ * @param analyzer the analyzer that will be used to split <code>text</code> into chunks* @param text text to highlight terms in* @param fieldName Name of field used to influence analyzer's tokenization policy- ** @return highlighted text fragment or null if no terms found- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/- public final String getBestFragment(Analyzer analyzer, String fieldName,String text)- throws IOException, InvalidTokenOffsetsException- {+ public final String getBestFragment(Analyzer analyzer, String fieldName, String text)+ throws IOException, InvalidTokenOffsetsException {TokenStream tokenStream = analyzer.tokenStream(fieldName, text);return getBestFragment(tokenStream, text);}/**- * Highlights chosen terms in a text, extracting the most relevant section.- * The document text is analysed in chunks to record hit statistics- * across the document. After accumulating stats, the fragment with the highest score- * is returned+ * Highlights chosen terms in a text, extracting the most relevant section. The document text is+ * analysed in chunks to record hit statistics across the document. After accumulating stats, the+ * fragment with the highest score is returned*- * @param tokenStream a stream of tokens identified in the text parameter, including offset information.- * This is typically produced by an analyzer re-parsing a document's- * text. Some work may be done on retrieving TokenStreams more efficiently- * by adding support for storing original text position data in the Lucene- * index but this support is not currently available (as of Lucene 1.4 rc2).+ * @param tokenStream a stream of tokens identified in the text parameter, including offset+ * information. This is typically produced by an analyzer re-parsing a document's text. Some+ * work may be done on retrieving TokenStreams more efficiently by adding support for storing+ * original text position data in the Lucene index but this support is not currently available+ * (as of Lucene 1.4 rc2).* @param text text to highlight terms in- ** @return highlighted text fragment or null if no terms found- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/public final String getBestFragment(TokenStream tokenStream, String text)- throws IOException, InvalidTokenOffsetsException- {- String[] results = getBestFragments(tokenStream,text, 1);- if (results.length > 0)- {+ throws IOException, InvalidTokenOffsetsException {+ String[] results = getBestFragments(tokenStream, text, 1);+ if (results.length > 0) {return results[0];}return null;}/**- * Highlights chosen terms in a text, extracting the most relevant sections.- * This is a convenience method that calls- * {@link #getBestFragments(TokenStream, String, int)}- *- * @param analyzer the analyzer that will be used to split <code>text</code>- * into chunks- * @param fieldName the name of the field being highlighted (used by analyzer)- * @param text text to highlight terms in- * @param maxNumFragments the maximum number of fragments.+ * Highlights chosen terms in a text, extracting the most relevant sections. This is a convenience+ * method that calls {@link #getBestFragments(TokenStream, String, int)}*+ * @param analyzer the analyzer that will be used to split <code>text</code> into chunks+ * @param fieldName the name of the field being highlighted (used by analyzer)+ * @param text text to highlight terms in+ * @param maxNumFragments the maximum number of fragments.* @return highlighted text fragments (between 0 and maxNumFragments number of fragments)- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/public final String[] getBestFragments(- Analyzer analyzer,- String fieldName,- String text,- int maxNumFragments)- throws IOException, InvalidTokenOffsetsException- {+ Analyzer analyzer, String fieldName, String text, int maxNumFragments)+ throws IOException, InvalidTokenOffsetsException {TokenStream tokenStream = analyzer.tokenStream(fieldName, text);return getBestFragments(tokenStream, text, maxNumFragments);}/**- * Highlights chosen terms in a text, extracting the most relevant sections.- * The document text is analysed in chunks to record hit statistics- * across the document. After accumulating stats, the fragments with the highest scores- * are returned as an array of strings in order of score (contiguous fragments are merged into- * one in their original order to improve readability)- *- * @param text text to highlight terms in- * @param maxNumFragments the maximum number of fragments.+ * Highlights chosen terms in a text, extracting the most relevant sections. The document text is+ * analysed in chunks to record hit statistics across the document. After accumulating stats, the+ * fragments with the highest scores are returned as an array of strings in order of score+ * (contiguous fragments are merged into one in their original order to improve readability)*+ * @param text text to highlight terms in+ * @param maxNumFragments the maximum number of fragments.* @return highlighted text fragments (between 0 and maxNumFragments number of fragments)- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/- public final String[] getBestFragments(- TokenStream tokenStream,- String text,- int maxNumFragments)- throws IOException, InvalidTokenOffsetsException- {- maxNumFragments = Math.max(1, maxNumFragments); //sanity check+ public final String[] getBestFragments(TokenStream tokenStream, String text, int maxNumFragments)+ throws IOException, InvalidTokenOffsetsException {+ maxNumFragments = Math.max(1, maxNumFragments); // sanity check- TextFragment[] frag =getBestTextFragments(tokenStream,text, true,maxNumFragments);+ TextFragment[] frag = getBestTextFragments(tokenStream, text, true, maxNumFragments);- //Get text+ // Get textArrayList<String> fragTexts = new ArrayList<>();- for (int i = 0; i < frag.length; i++)- {- if ((frag[i] != null) && (frag[i].getScore() > 0))- {+ for (int i = 0; i < frag.length; i++) {+ if ((frag[i] != null) && (frag[i].getScore() > 0)) {fragTexts.add(frag[i].toString());}}return fragTexts.toArray(new String[0]);}-/**- * Low level api to get the most relevant (formatted) sections of the document.- * This method has been made public to allow visibility of score information held in TextFragment objects.- * Thanks to Jason Calabrese for help in redefining the interface.+ * Low level api to get the most relevant (formatted) sections of the document. This method has+ * been made public to allow visibility of score information held in TextFragment objects. Thanks+ * to Jason Calabrese for help in redefining the interface.+ ** @throws IOException If there is a low-level I/O error- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/public final TextFragment[] getBestTextFragments(- TokenStream tokenStream,- String text,- boolean mergeContiguousFragments,- int maxNumFragments)- throws IOException, InvalidTokenOffsetsException- {+ TokenStream tokenStream, String text, boolean mergeContiguousFragments, int maxNumFragments)+ throws IOException, InvalidTokenOffsetsException {ArrayList<TextFragment> docFrags = new ArrayList<>();- StringBuilder newText=new StringBuilder();+ StringBuilder newText = new StringBuilder();CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);- TextFragment currentFrag = new TextFragment(newText,newText.length(), docFrags.size());+ TextFragment currentFrag = new TextFragment(newText, newText.length(), docFrags.size());if (fragmentScorer instanceof QueryScorer) {((QueryScorer) fragmentScorer).setMaxDocCharsToAnalyze(maxDocCharsToAnalyze);}TokenStream newStream = fragmentScorer.init(tokenStream);- if(newStream != null) {+ if (newStream != null) {tokenStream = newStream;}fragmentScorer.startFragment(currentFrag);@@ -207,8 +179,7 @@ public class HighlighterFragmentQueue fragQueue = new FragmentQueue(maxNumFragments);- try- {+ try {String tokenText;int startOffset;@@ -216,42 +187,39 @@ public class Highlighterint lastEndOffset = 0;textFragmenter.start(text, tokenStream);- TokenGroup tokenGroup=new TokenGroup(tokenStream);+ TokenGroup tokenGroup = new TokenGroup(tokenStream);tokenStream.reset();- for (boolean next = tokenStream.incrementToken(); next && (offsetAtt.startOffset()< maxDocCharsToAnalyze);- next = tokenStream.incrementToken())- {- if( (offsetAtt.endOffset()>text.length())- ||- (offsetAtt.startOffset()>text.length())- )- {- throw new InvalidTokenOffsetsException("Token "+ termAtt.toString()- +" exceeds length of provided text sized "+text.length());+ for (boolean next = tokenStream.incrementToken();+ next && (offsetAtt.startOffset() < maxDocCharsToAnalyze);+ next = tokenStream.incrementToken()) {+ if ((offsetAtt.endOffset() > text.length()) || (offsetAtt.startOffset() > text.length())) {+ throw new InvalidTokenOffsetsException(+ "Token "+ + termAtt.toString()+ + " exceeds length of provided text sized "+ + text.length());}- if((tokenGroup.getNumTokens() >0)&&(tokenGroup.isDistinct()))- {- //the current token is distinct from previous tokens -+ if ((tokenGroup.getNumTokens() > 0) && (tokenGroup.isDistinct())) {+ // the current token is distinct from previous tokens -// markup the cached token group infostartOffset = tokenGroup.getStartOffset();endOffset = tokenGroup.getEndOffset();tokenText = text.substring(startOffset, endOffset);- String markedUpText=formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);- //store any whitespace etc from between this and last group+ String markedUpText = formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);+ // store any whitespace etc from between this and last groupif (startOffset > lastEndOffset)newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));newText.append(markedUpText);- lastEndOffset=Math.max(endOffset, lastEndOffset);+ lastEndOffset = Math.max(endOffset, lastEndOffset);tokenGroup.clear();- //check if current token marks the start of a new fragment- if(textFragmenter.isNewFragment())- {+ // check if current token marks the start of a new fragment+ if (textFragmenter.isNewFragment()) {currentFrag.setScore(fragmentScorer.getFragmentScore());- //record stats for a new fragment+ // record stats for a new fragmentcurrentFrag.textEndPos = newText.length();- currentFrag =new TextFragment(newText, newText.length(), docFrags.size());+ currentFrag = new TextFragment(newText, newText.length(), docFrags.size());fragmentScorer.startFragment(currentFrag);docFrags.add(currentFrag);}@@ -259,48 +227,44 @@ public class HighlightertokenGroup.addToken(fragmentScorer.getTokenScore());-// if(lastEndOffset>maxDocBytesToAnalyze)-// {-// break;-// }+ // if(lastEndOffset>maxDocBytesToAnalyze)+ // {+ // break;+ // }}currentFrag.setScore(fragmentScorer.getFragmentScore());- if(tokenGroup.getNumTokens() >0)- {- //flush the accumulated text (same code as in above loop)+ if (tokenGroup.getNumTokens() > 0) {+ // flush the accumulated text (same code as in above loop)startOffset = tokenGroup.getStartOffset();endOffset = tokenGroup.getEndOffset();tokenText = text.substring(startOffset, endOffset);- String markedUpText=formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);- //store any whitespace etc from between this and last group+ String markedUpText = formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);+ // store any whitespace etc from between this and last groupif (startOffset > lastEndOffset)newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));newText.append(markedUpText);- lastEndOffset=Math.max(lastEndOffset,endOffset);+ lastEndOffset = Math.max(lastEndOffset, endOffset);}- //Test what remains of the original text beyond the point where we stopped analyzing+ // Test what remains of the original text beyond the point where we stopped analyzingif (-// if there is text beyond the last token considered..- (lastEndOffset < text.length())+ // if there is text beyond the last token considered..+ (lastEndOffset < text.length())&&-// and that text is not too large...- (text.length()<= maxDocCharsToAnalyze)- )- {- //append it to the last fragment+ // and that text is not too large...+ (text.length() <= maxDocCharsToAnalyze)) {+ // append it to the last fragmentnewText.append(encoder.encodeText(text.substring(lastEndOffset)));}currentFrag.textEndPos = newText.length();- //sort the most relevant sections of the text- for (Iterator<TextFragment> i = docFrags.iterator(); i.hasNext();)- {+ // sort the most relevant sections of the text+ for (Iterator<TextFragment> i = docFrags.iterator(); i.hasNext(); ) {currentFrag = i.next();- //If you are running with a version of Lucene before 11th Sept 03+ // If you are running with a version of Lucene before 11th Sept 03// you do not have PriorityQueue.insert() - so uncomment the code below/*if (currentFrag.getScore() >= minScore)@@ -315,83 +279,66 @@ public class Highlighter}*/- //The above code caused a problem as a result of Christoph Goller's 11th Sept 03- //fix to PriorityQueue. The correct method to use here is the new "insert" method+ // The above code caused a problem as a result of Christoph Goller's 11th Sept 03+ // fix to PriorityQueue. The correct method to use here is the new "insert" method// USE ABOVE CODE IF THIS DOES NOT COMPILE!fragQueue.insertWithOverflow(currentFrag);}- //return the most relevant fragments+ // return the most relevant fragmentsTextFragment frag[] = new TextFragment[fragQueue.size()];- for (int i = frag.length - 1; i >= 0; i--)- {+ for (int i = frag.length - 1; i >= 0; i--) {frag[i] = fragQueue.pop();}- //merge any contiguous fragments to improve readability- if(mergeContiguousFragments)- {+ // merge any contiguous fragments to improve readability+ if (mergeContiguousFragments) {mergeContiguousFragments(frag);ArrayList<TextFragment> fragTexts = new ArrayList<>();- for (int i = 0; i < frag.length; i++)- {- if ((frag[i] != null) && (frag[i].getScore() > 0))- {+ for (int i = 0; i < frag.length; i++) {+ if ((frag[i] != null) && (frag[i].getScore() > 0)) {fragTexts.add(frag[i]);}}- frag= fragTexts.toArray(new TextFragment[0]);+ frag = fragTexts.toArray(new TextFragment[0]);}return frag;- }- finally- {- if (tokenStream != null)- {- try- {+ } finally {+ if (tokenStream != null) {+ try {tokenStream.end();tokenStream.close();- }- catch (Exception e)- {+ } catch (Exception e) {}}}}-- /** Improves readability of a score-sorted list of TextFragments by merging any fragments- * that were contiguous in the original text into one larger fragment with the correct order.- * This will leave a "null" in the array entry for the lesser scored fragment.+ /**+ * Improves readability of a score-sorted list of TextFragments by merging any fragments that were+ * contiguous in the original text into one larger fragment with the correct order. This will+ * leave a "null" in the array entry for the lesser scored fragment.** @param frag An array of document fragments in descending score*/- private void mergeContiguousFragments(TextFragment[] frag)- {+ private void mergeContiguousFragments(TextFragment[] frag) {boolean mergingStillBeingDone;if (frag.length > 1)- do- {- mergingStillBeingDone = false; //initialise loop control flag- //for each fragment, scan other frags looking for contiguous blocks- for (int i = 0; i < frag.length; i++)- {- if (frag[i] == null)- {+ do {+ mergingStillBeingDone = false; // initialise loop control flag+ // for each fragment, scan other frags looking for contiguous blocks+ for (int i = 0; i < frag.length; i++) {+ if (frag[i] == null) {continue;}- //merge any contiguous blocks- for (int x = 0; x < frag.length; x++)- {- if (frag[x] == null)- {+ // merge any contiguous blocks+ for (int x = 0; x < frag.length; x++) {+ if (frag[x] == null) {continue;}- if (frag[i] == null)- {+ if (frag[i] == null) {break;}TextFragment frag1 = null;@@ -400,32 +347,24 @@ public class Highlighterint frag2Num = 0;int bestScoringFragNum;int worstScoringFragNum;- //if blocks are contiguous....- if (frag[i].follows(frag[x]))- {+ // if blocks are contiguous....+ if (frag[i].follows(frag[x])) {frag1 = frag[x];frag1Num = x;frag2 = frag[i];frag2Num = i;+ } else if (frag[x].follows(frag[i])) {+ frag1 = frag[i];+ frag1Num = i;+ frag2 = frag[x];+ frag2Num = x;}- else- if (frag[x].follows(frag[i]))- {- frag1 = frag[i];- frag1Num = i;- frag2 = frag[x];- frag2Num = x;- }- //merging required..- if (frag1 != null)- {- if (frag1.getScore() > frag2.getScore())- {+ // merging required..+ if (frag1 != null) {+ if (frag1.getScore() > frag2.getScore()) {bestScoringFragNum = frag1Num;worstScoringFragNum = frag2Num;- }- else- {+ } else {bestScoringFragNum = frag2Num;worstScoringFragNum = frag1Num;}@@ -436,38 +375,29 @@ public class Highlighter}}}- }- while (mergingStillBeingDone);+ } while (mergingStillBeingDone);}-/**- * Highlights terms in the text , extracting the most relevant sections- * and concatenating the chosen fragments with a separator (typically "...").- * The document text is analysed in chunks to record hit statistics- * across the document. After accumulating stats, the fragments with the highest scores- * are returned in order as "separator" delimited strings.- *- * @param text text to highlight terms in- * @param maxNumFragments the maximum number of fragments.- * @param separator the separator used to intersperse the document fragments (typically "...")+ * Highlights terms in the text , extracting the most relevant sections and concatenating the+ * chosen fragments with a separator (typically "..."). The document text is analysed in chunks to+ * record hit statistics across the document. After accumulating stats, the fragments with the+ * highest scores are returned in order as "separator" delimited strings.*+ * @param text text to highlight terms in+ * @param maxNumFragments the maximum number of fragments.+ * @param separator the separator used to intersperse the document fragments (typically "...")* @return highlighted text- * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided+ * text's length*/public final String getBestFragments(- TokenStream tokenStream,- String text,- int maxNumFragments,- String separator)- throws IOException, InvalidTokenOffsetsException- {- String sections[] = getBestFragments(tokenStream,text, maxNumFragments);+ TokenStream tokenStream, String text, int maxNumFragments, String separator)+ throws IOException, InvalidTokenOffsetsException {+ String sections[] = getBestFragments(tokenStream, text, maxNumFragments);StringBuilder result = new StringBuilder();- for (int i = 0; i < sections.length; i++)- {- if (i > 0)- {+ for (int i = 0; i < sections.length; i++) {+ if (i > 0) {result.append(separator);}result.append(sections[i]);@@ -483,26 +413,20 @@ public class Highlighterthis.maxDocCharsToAnalyze = maxDocCharsToAnalyze;}- public Fragmenter getTextFragmenter()- {+ public Fragmenter getTextFragmenter() {return textFragmenter;}- public void setTextFragmenter(Fragmenter fragmenter)- {+ public void setTextFragmenter(Fragmenter fragmenter) {textFragmenter = Objects.requireNonNull(fragmenter);}- /**- * @return Object used to score each text fragment- */- public Scorer getFragmentScorer()- {+ /** @return Object used to score each text fragment */+ public Scorer getFragmentScorer() {return fragmentScorer;}- public void setFragmentScorer(Scorer scorer)- {+ public void setFragmentScorer(Scorer scorer) {fragmentScorer = Objects.requireNonNull(scorer);}@@ -518,7 +442,7 @@ public class Highlighter* Throws an IllegalArgumentException with the provided message if 'argument' is null.** @param argument the argument to be null-checked- * @param message the message of the exception thrown if argument == null+ * @param message the message of the exception thrown if argument == null*/private static void ensureArgumentNotNull(Object argument, String message) {if (argument == null) {@@ -526,20 +450,15 @@ public class Highlighter}}- static class FragmentQueue extends PriorityQueue<TextFragment>- {- FragmentQueue(int size)- {+ static class FragmentQueue extends PriorityQueue<TextFragment> {+ FragmentQueue(int size) {super(size);}@Override- public final boolean lessThan(TextFragment fragA, TextFragment fragB)- {- if (fragA.getScore() == fragB.getScore())- return fragA.fragNum > fragB.fragNum;- else- return fragA.getScore() < fragB.getScore();+ public final boolean lessThan(TextFragment fragA, TextFragment fragB) {+ if (fragA.getScore() == fragB.getScore()) return fragA.fragNum > fragB.fragNum;+ else return fragA.getScore() < fragB.getScore();}}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.javaindex fe353a22c49..8c7f9613846 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/InvalidTokenOffsetsException.java@@ -15,16 +15,10 @@* limitations under the License.*/package org.apache.lucene.search.highlight;-/**- * Exception thrown if TokenStream Tokens are incompatible with provided text- *- */-public class InvalidTokenOffsetsException extends Exception-{+/** Exception thrown if TokenStream Tokens are incompatible with provided text */+public class InvalidTokenOffsetsException extends Exception {- public InvalidTokenOffsetsException(String message)- {+ public InvalidTokenOffsetsException(String message) {super(message);}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/LimitTokenOffsetFilter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/LimitTokenOffsetFilter.javaindex 39763579ca3..fc27a034e66 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/LimitTokenOffsetFilter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/LimitTokenOffsetFilter.java@@ -17,14 +17,13 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;/**- * This is a simplified version of org.apache.lucene.analysis.miscellaneous.LimitTokenOffsetFilter to prevent- * a dependency on analysis-common.jar.+ * This is a simplified version of org.apache.lucene.analysis.miscellaneous.LimitTokenOffsetFilter+ * to prevent a dependency on analysis-common.jar.*/final class LimitTokenOffsetFilter extends TokenFilter {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.javaindex e6052219605..73736e9bd7f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/NullFragmenter.java@@ -15,20 +15,19 @@* limitations under the License.*/package org.apache.lucene.search.highlight;+import org.apache.lucene.analysis.TokenStream;/**- * {@link Fragmenter} implementation which does not fragment the text.- * This is useful for highlighting the entire content of a document or field.+ * {@link Fragmenter} implementation which does not fragment the text. This is useful for+ * highlighting the entire content of a document or field.*/public class NullFragmenter implements Fragmenter {@Override- public void start(String s, TokenStream tokenStream) {- }+ public void start(String s, TokenStream tokenStream) {}@Overridepublic boolean isNewFragment() {return false;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.javaindex 23557b54ccd..ba598f7ed38 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java@@ -17,26 +17,22 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;-/**- * This TokenFilter limits the number of tokens while indexing by adding up the- * current offset.- */+/** This TokenFilter limits the number of tokens while indexing by adding up the current offset. */public final class OffsetLimitTokenFilter extends TokenFilter {-+private int offsetCount;private OffsetAttribute offsetAttrib = getAttribute(OffsetAttribute.class);private int offsetLimit;-+public OffsetLimitTokenFilter(TokenStream input, int offsetLimit) {super(input);this.offsetLimit = offsetLimit;}-+@Overridepublic boolean incrementToken() throws IOException {if (offsetCount < offsetLimit && input.incrementToken()) {@@ -46,11 +42,10 @@ public final class OffsetLimitTokenFilter extends TokenFilter {}return false;}-+@Overridepublic void reset() throws IOException {super.reset();offsetCount = 0;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.javaindex ac7a9d02be2..c08abc7435f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/PositionSpan.java@@ -17,6 +17,7 @@package org.apache.lucene.search.highlight;/*** Utility class to record Positions Spans+ ** @lucene.internal*/public class PositionSpan {@@ -27,4 +28,4 @@ public class PositionSpan {this.start = start;this.end = end;}-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.javaindex 5452fc7e830..977dabb3c19 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java@@ -21,7 +21,6 @@ import java.util.HashMap;import java.util.HashSet;import java.util.Map;import java.util.Set;-import org.apache.lucene.analysis.CachingTokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;@@ -32,15 +31,14 @@ import org.apache.lucene.search.Query;import org.apache.lucene.search.spans.SpanQuery;/**- * {@link Scorer} implementation which scores text fragments by the number of- * unique query terms found. This class converts appropriate {@link Query}s to- * {@link SpanQuery}s and attempts to score only those terms that participated in- * generating the 'hit' on the document.+ * {@link Scorer} implementation which scores text fragments by the number of unique query terms+ * found. This class converts appropriate {@link Query}s to {@link SpanQuery}s and attempts to score+ * only those terms that participated in generating the 'hit' on the document.*/public class QueryScorer implements Scorer {private float totalScore;private Set<String> foundTerms;- private Map<String,WeightedSpanTerm> fieldWeightedSpanTerms;+ private Map<String, WeightedSpanTerm> fieldWeightedSpanTerms;private float maxTermWeight;private int position = -1;private String defaultField;@@ -55,9 +53,7 @@ public class QueryScorer implements Scorer {private int maxCharsToAnalyze;private boolean usePayloads = false;- /**- * @param query Query to use for highlighting- */+ /** @param query Query to use for highlighting */public QueryScorer(Query query) {init(query, null, null, true);}@@ -79,7 +75,6 @@ public class QueryScorer implements Scorer {init(query, field, reader, true);}-/*** @param query to use for highlighting* @param reader {@link IndexReader} to use for quasi tf/idf scoring@@ -90,25 +85,20 @@ public class QueryScorer implements Scorer {init(query, field, reader, true);}- /**- * @param defaultField - The default field for queries with the field name unspecified- */+ /** @param defaultField - The default field for queries with the field name unspecified */public QueryScorer(Query query, String field, String defaultField) {this.defaultField = defaultField;init(query, field, null, true);}- /**- * @param weightedTerms an array of pre-created {@link WeightedSpanTerm}s- */+ /** @param weightedTerms an array of pre-created {@link WeightedSpanTerm}s */public QueryScorer(WeightedSpanTerm[] weightedTerms) {this.fieldWeightedSpanTerms = new HashMap<>(weightedTerms.length);for (int i = 0; i < weightedTerms.length; i++) {WeightedSpanTerm existingTerm = fieldWeightedSpanTerms.get(weightedTerms[i].term);- if ((existingTerm == null) ||- (existingTerm.weight < weightedTerms[i].weight)) {+ if ((existingTerm == null) || (existingTerm.weight < weightedTerms[i].weight)) {// if a term is defined more than once, always use the highest// scoring weightfieldWeightedSpanTerms.put(weightedTerms[i].term, weightedTerms[i]);@@ -129,9 +119,8 @@ public class QueryScorer implements Scorer {}/**- *- * @return The highest weighted term (useful for passing to- * GradientFormatter to set top end of coloring scale).+ * @return The highest weighted term (useful for passing to GradientFormatter to set top end of+ * coloring scale).*/public float getMaxTermWeight() {return maxTermWeight;@@ -150,13 +139,11 @@ public class QueryScorer implements Scorer {WeightedSpanTerm weightedSpanTerm;- if ((weightedSpanTerm = fieldWeightedSpanTerms.get(- termText)) == null) {+ if ((weightedSpanTerm = fieldWeightedSpanTerms.get(termText)) == null) {return 0;}- if (weightedSpanTerm.positionSensitive &&- !weightedSpanTerm.checkPosition(position)) {+ if (weightedSpanTerm.positionSensitive && !weightedSpanTerm.checkPosition(position)) {return 0;}@@ -179,18 +166,18 @@ public class QueryScorer implements Scorer {position = -1;termAtt = tokenStream.addAttribute(CharTermAttribute.class);posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);- if(!skipInitExtractor) {- if(fieldWeightedSpanTerms != null) {+ if (!skipInitExtractor) {+ if (fieldWeightedSpanTerms != null) {fieldWeightedSpanTerms.clear();}return initExtractor(tokenStream);}return null;}-+/**- * Retrieve the {@link WeightedSpanTerm} for the specified token. Useful for passing- * Span information to a {@link Fragmenter}.+ * Retrieve the {@link WeightedSpanTerm} for the specified token. Useful for passing Span+ * information to a {@link Fragmenter}.** @param token to get {@link WeightedSpanTerm} for* @return WeightedSpanTerm for token@@ -199,15 +186,14 @@ public class QueryScorer implements Scorer {return fieldWeightedSpanTerms.get(token);}- /**- */+ /** */private void init(Query query, String field, IndexReader reader, boolean expandMultiTermQuery) {this.reader = reader;this.expandMultiTermQuery = expandMultiTermQuery;this.query = query;this.field = field;}-+private TokenStream initExtractor(TokenStream tokenStream) throws IOException {WeightedSpanTermExtractor qse = newTermExtractor(defaultField);qse.setMaxDocCharsToAnalyze(maxCharsToAnalyze);@@ -215,19 +201,18 @@ public class QueryScorer implements Scorer {qse.setWrapIfNotCachingTokenFilter(wrapToCaching);qse.setUsePayloads(usePayloads);if (reader == null) {- this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query, 1f,- tokenStream, field);+ this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query, 1f, tokenStream, field);} else {- this.fieldWeightedSpanTerms = qse.getWeightedSpanTermsWithScores(query, 1f,- tokenStream, field, reader);+ this.fieldWeightedSpanTerms =+ qse.getWeightedSpanTermsWithScores(query, 1f, tokenStream, field, reader);}- if(qse.isCachedTokenStream()) {+ if (qse.isCachedTokenStream()) {return qse.getTokenStream();}-+return null;}-+protected WeightedSpanTermExtractor newTermExtractor(String defaultField) {return new WeightedSpanTermExtractor(defaultField);}@@ -242,18 +227,16 @@ public class QueryScorer implements Scorer {foundTerms = new HashSet<>();totalScore = 0;}-- /**- * @return true if multi-term queries should be expanded- */++ /** @return true if multi-term queries should be expanded */public boolean isExpandMultiTermQuery() {return expandMultiTermQuery;}/**- * Controls whether or not multi-term queries are expanded- * against a {@link MemoryIndex} {@link IndexReader}.- *+ * Controls whether or not multi-term queries are expanded against a {@link MemoryIndex} {@link+ * IndexReader}.+ ** @param expandMultiTermQuery true if multi-term queries should be expanded*/public void setExpandMultiTermQuery(boolean expandMultiTermQuery) {@@ -261,8 +244,9 @@ public class QueryScorer implements Scorer {}/**- * Whether or not we should capture payloads in {@link MemoryIndex} at each position so that queries can access them.- * This does not apply to term vector based TokenStreams, which support payloads only when the term vector has them.+ * Whether or not we should capture payloads in {@link MemoryIndex} at each position so that+ * queries can access them. This does not apply to term vector based TokenStreams, which support+ * payloads only when the term vector has them.*/public boolean isUsePayloads() {return usePayloads;@@ -273,12 +257,10 @@ public class QueryScorer implements Scorer {}/**- * By default, {@link TokenStream}s that are not of the type- * {@link CachingTokenFilter} are wrapped in a {@link CachingTokenFilter} to- * ensure an efficient reset - if you are already using a different caching- * {@link TokenStream} impl and you don't want it to be wrapped, set this to- * false. Note that term-vector based tokenstreams are detected and won't be- * wrapped either.+ * By default, {@link TokenStream}s that are not of the type {@link CachingTokenFilter} are+ * wrapped in a {@link CachingTokenFilter} to ensure an efficient reset - if you are already using+ * a different caching {@link TokenStream} impl and you don't want it to be wrapped, set this to+ * false. Note that term-vector based tokenstreams are detected and won't be wrapped either.*/public void setWrapIfNotCachingTokenFilter(boolean wrap) {this.wrapToCaching = wrap;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.javaindex 0b5738f492a..53617b37f45 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java@@ -20,7 +20,6 @@ import java.io.IOException;import java.util.HashSet;import java.util.Set;import java.util.function.Predicate;-import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.MultiReader;import org.apache.lucene.index.Term;@@ -31,17 +30,15 @@ import org.apache.lucene.search.Query;import org.apache.lucene.search.QueryVisitor;/**- * Utility class used to extract the terms used in a query, plus any weights.- * This class will not find terms for MultiTermQuery, TermRangeQuery and PrefixQuery classes- * so the caller must pass a rewritten query (see Query.rewrite) to obtain a list of- * expanded terms.- *+ * Utility class used to extract the terms used in a query, plus any weights. This class will not+ * find terms for MultiTermQuery, TermRangeQuery and PrefixQuery classes so the caller must pass a+ * rewritten query (see Query.rewrite) to obtain a list of expanded terms.*/-public final class QueryTermExtractor-{+public final class QueryTermExtractor {/** for term extraction */private static final IndexSearcher EMPTY_INDEXSEARCHER;+static {try {IndexReader emptyReader = new MultiReader();@@ -55,50 +52,45 @@ public final class QueryTermExtractor/*** Extracts all terms texts of a given Query into an array of WeightedTerms*- * @param query Query to extract term texts from+ * @param query Query to extract term texts from* @return an array of the terms used in a query, plus their weights.*/- public static final WeightedTerm[] getTerms(Query query)- {- return getTerms(query,false);+ public static final WeightedTerm[] getTerms(Query query) {+ return getTerms(query, false);}/*** Extracts all terms texts of a given Query into an array of WeightedTerms*- * @param query Query to extract term texts from- * @param reader used to compute IDF which can be used to a) score selected fragments better- * b) use graded highlights eg changing intensity of font color+ * @param query Query to extract term texts from+ * @param reader used to compute IDF which can be used to a) score selected fragments better b)+ * use graded highlights eg changing intensity of font color* @param fieldName the field on which Inverse Document Frequency (IDF) calculations are based* @return an array of the terms used in a query, plus their weights.*/- public static final WeightedTerm[] getIdfWeightedTerms(Query query, IndexReader reader, String fieldName)- {- WeightedTerm[] terms=getTerms(query,false, fieldName);- int totalNumDocs=reader.maxDoc();- for (int i = 0; i < terms.length; i++)- {- try- {- int docFreq=reader.docFreq(new Term(fieldName,terms[i].term));- //IDF algorithm taken from ClassicSimilarity class- float idf=(float)(Math.log(totalNumDocs/(double)(docFreq+1)) + 1.0);- terms[i].weight*=idf;- }- catch (IOException e)- {- //ignore- }- }+ public static final WeightedTerm[] getIdfWeightedTerms(+ Query query, IndexReader reader, String fieldName) {+ WeightedTerm[] terms = getTerms(query, false, fieldName);+ int totalNumDocs = reader.maxDoc();+ for (int i = 0; i < terms.length; i++) {+ try {+ int docFreq = reader.docFreq(new Term(fieldName, terms[i].term));+ // IDF algorithm taken from ClassicSimilarity class+ float idf = (float) (Math.log(totalNumDocs / (double) (docFreq + 1)) + 1.0);+ terms[i].weight *= idf;+ } catch (IOException e) {+ // ignore+ }+ }return terms;}/*** Extracts all terms texts of a given Query into an array of WeightedTerms*- * @param query Query to extract term texts from+ * @param query Query to extract term texts from* @param prohibited <code>true</code> to extract "prohibited" terms, too- * @param fieldName The fieldName used to filter query terms+ * @param fieldName The fieldName used to filter query terms* @return an array of the terms used in a query, plus their weights.*/public static WeightedTerm[] getTerms(Query query, boolean prohibited, String fieldName) {@@ -111,13 +103,12 @@ public final class QueryTermExtractor/*** Extracts all terms texts of a given Query into an array of WeightedTerms*- * @param query Query to extract term texts from+ * @param query Query to extract term texts from* @param prohibited <code>true</code> to extract "prohibited" terms, too* @return an array of the terms used in a query, plus their weights.*/- public static final WeightedTerm[] getTerms(Query query, boolean prohibited)- {- return getTerms(query,prohibited,null);+ public static final WeightedTerm[] getTerms(Query query, boolean prohibited) {+ return getTerms(query, prohibited, null);}private static class BoostedTermExtractor extends QueryVisitor {@@ -127,8 +118,11 @@ public final class QueryTermExtractorfinal boolean includeProhibited;final Predicate<String> fieldSelector;- private BoostedTermExtractor(float boost, Set<WeightedTerm> terms, boolean includeProhibited,- Predicate<String> fieldSelector) {+ private BoostedTermExtractor(+ float boost,+ Set<WeightedTerm> terms,+ boolean includeProhibited,+ Predicate<String> fieldSelector) {this.boost = boost;this.terms = terms;this.includeProhibited = includeProhibited;@@ -150,7 +144,7 @@ public final class QueryTermExtractor@Overridepublic QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) {if (parent instanceof BoostQuery) {- float newboost = boost * ((BoostQuery)parent).getBoost();+ float newboost = boost * ((BoostQuery) parent).getBoost();return new BoostedTermExtractor(newboost, terms, includeProhibited, fieldSelector);}if (occur == BooleanClause.Occur.MUST_NOT && includeProhibited == false) {@@ -158,7 +152,5 @@ public final class QueryTermExtractor}return this;}-}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.javaindex 229b42c369a..0d668f12c33 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java@@ -18,43 +18,40 @@ package org.apache.lucene.search.highlight;import java.util.HashMap;import java.util.HashSet;-import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.index.IndexReader;import org.apache.lucene.search.Query;/**- * {@link Scorer} implementation which scores text fragments by the number of- * unique query terms found. This class uses the {@link QueryTermExtractor}- * class to process determine the query terms and their boosts to be used.+ * {@link Scorer} implementation which scores text fragments by the number of unique query terms+ * found. This class uses the {@link QueryTermExtractor} class to process determine the query terms+ * and their boosts to be used.*/// TODO: provide option to boost score of fragments near beginning of document// based on fragment.getFragNum()public class QueryTermScorer implements Scorer {-+TextFragment currentTextFragment = null;HashSet<String> uniqueTermsInFragment;float totalScore = 0;float maxTermWeight = 0;- private HashMap<String,WeightedTerm> termsToFind;+ private HashMap<String, WeightedTerm> termsToFind;private CharTermAttribute termAtt;/**- *- * @param query a Lucene query (ideally rewritten using query.rewrite before- * being passed to this class and the searcher)+ * @param query a Lucene query (ideally rewritten using query.rewrite before being passed to this+ * class and the searcher)*/public QueryTermScorer(Query query) {this(QueryTermExtractor.getTerms(query));}/**- *- * @param query a Lucene query (ideally rewritten using query.rewrite before- * being passed to this class and the searcher)+ * @param query a Lucene query (ideally rewritten using query.rewrite before being passed to this+ * class and the searcher)* @param fieldName the Field name which is used to match Query terms*/public QueryTermScorer(Query query, String fieldName) {@@ -62,14 +59,11 @@ public class QueryTermScorer implements Scorer {}/**- *- * @param query a Lucene query (ideally rewritten using query.rewrite before- * being passed to this class and the searcher)- * @param reader used to compute IDF which can be used to a) score selected- * fragments better b) use graded highlights eg set font color- * intensity- * @param fieldName the field on which Inverse Document Frequency (IDF)- * calculations are based+ * @param query a Lucene query (ideally rewritten using query.rewrite before being passed to this+ * class and the searcher)+ * @param reader used to compute IDF which can be used to a) score selected fragments better b)+ * use graded highlights eg set font color intensity+ * @param fieldName the field on which Inverse Document Frequency (IDF) calculations are based*/public QueryTermScorer(Query query, IndexReader reader, String fieldName) {this(QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName));@@ -78,10 +72,8 @@ public class QueryTermScorer implements Scorer {public QueryTermScorer(WeightedTerm[] weightedTerms) {termsToFind = new HashMap<>();for (int i = 0; i < weightedTerms.length; i++) {- WeightedTerm existingTerm = termsToFind- .get(weightedTerms[i].term);- if ((existingTerm == null)- || (existingTerm.weight < weightedTerms[i].weight)) {+ WeightedTerm existingTerm = termsToFind.get(weightedTerms[i].term);+ if ((existingTerm == null) || (existingTerm.weight < weightedTerms[i].weight)) {// if a term is defined more than once, always use the highest scoring// weighttermsToFind.put(weightedTerms[i].term, weightedTerms[i]);@@ -101,7 +93,7 @@ public class QueryTermScorer implements Scorer {/** (non-Javadoc)- *+ ** @see* org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache* .lucene.search.highlight.TextFragment)@@ -111,10 +103,8 @@ public class QueryTermScorer implements Scorer {uniqueTermsInFragment = new HashSet<>();currentTextFragment = newFragment;totalScore = 0;-}-/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Scorer#getTokenScore()*/@@ -135,7 +125,6 @@ public class QueryTermScorer implements Scorer {return queryTerm.getWeight();}-/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()*/@@ -146,7 +135,7 @@ public class QueryTermScorer implements Scorer {/** (non-Javadoc)- *+ ** @see* org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()*/@@ -155,9 +144,8 @@ public class QueryTermScorer implements Scorer {}/**- *- * @return The highest weighted term (useful for passing to GradientFormatter- * to set top end of coloring scale.+ * @return The highest weighted term (useful for passing to GradientFormatter to set top end of+ * coloring scale.*/public float getMaxTermWeight() {return maxTermWeight;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.javaindex fd8f48421ad..26c4c3218b9 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java@@ -17,50 +17,46 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.TokenStream;/**- * A Scorer is responsible for scoring a stream of tokens. These token scores- * can then be used to compute {@link TextFragment} scores.+ * A Scorer is responsible for scoring a stream of tokens. These token scores can then be used to+ * compute {@link TextFragment} scores.*/public interface Scorer {/**- * Called to init the Scorer with a {@link TokenStream}. You can grab references to- * the attributes you are interested in here and access them from {@link #getTokenScore()}.- *+ * Called to init the Scorer with a {@link TokenStream}. You can grab references to the attributes+ * you are interested in here and access them from {@link #getTokenScore()}.+ ** @param tokenStream the {@link TokenStream} that will be scored.- * @return either a {@link TokenStream} that the Highlighter should continue using (eg- * if you read the tokenSream in this method) or null to continue- * using the same {@link TokenStream} that was passed in.+ * @return either a {@link TokenStream} that the Highlighter should continue using (eg if you read+ * the tokenSream in this method) or null to continue using the same {@link TokenStream} that+ * was passed in.* @throws IOException If there is a low-level I/O error*/public TokenStream init(TokenStream tokenStream) throws IOException;/*** Called when a new fragment is started for consideration.- *+ ** @param newFragment the fragment that will be scored next*/public void startFragment(TextFragment newFragment);/**- * Called for each token in the current fragment. The {@link Highlighter} will- * increment the {@link TokenStream} passed to init on every call.- *- * @return a score which is passed to the {@link Highlighter} class to influence the- * mark-up of the text (this return value is NOT used to score the- * fragment)+ * Called for each token in the current fragment. The {@link Highlighter} will increment the+ * {@link TokenStream} passed to init on every call.+ *+ * @return a score which is passed to the {@link Highlighter} class to influence the mark-up of+ * the text (this return value is NOT used to score the fragment)*/public float getTokenScore();/**- * Called when the {@link Highlighter} has no more tokens for the current fragment -- * the Scorer returns the weighting it has derived for the most recent- * fragment, typically based on the results of {@link #getTokenScore()}.- *+ * Called when the {@link Highlighter} has no more tokens for the current fragment - the Scorer+ * returns the weighting it has derived for the most recent fragment, typically based on the+ * results of {@link #getTokenScore()}.*/public float getFragmentScore();-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.javaindex 4f25c3a7b33..c504918b87b 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java@@ -20,8 +20,8 @@ import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;/**- * {@link Fragmenter} implementation which breaks text up into same-size- * fragments with no concerns over spotting sentence boundaries.+ * {@link Fragmenter} implementation which breaks text up into same-size fragments with no concerns+ * over spotting sentence boundaries.*/public class SimpleFragmenter implements Fragmenter {private static final int DEFAULT_FRAGMENT_SIZE = 100;@@ -33,15 +33,11 @@ public class SimpleFragmenter implements Fragmenter {this(DEFAULT_FRAGMENT_SIZE);}- /**- *- * @param fragmentSize size in number of characters of each fragment- */+ /** @param fragmentSize size in number of characters of each fragment */public SimpleFragmenter(int fragmentSize) {this.fragmentSize = fragmentSize;}-/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Fragmenter#start(java.lang.String, org.apache.lucene.analysis.TokenStream)*/@@ -51,7 +47,6 @@ public class SimpleFragmenter implements Fragmenter {currentNumFrags = 1;}-/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Fragmenter#isNewFragment()*/@@ -64,18 +59,13 @@ public class SimpleFragmenter implements Fragmenter {return isNewFrag;}- /**- * @return size in number of characters of each fragment- */+ /** @return size in number of characters of each fragment */public int getFragmentSize() {return fragmentSize;}- /**- * @param size size in characters of each fragment- */+ /** @param size size in characters of each fragment */public void setFragmentSize(int size) {fragmentSize = size;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLEncoder.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLEncoder.javaindex 7e7630db2c1..393d895d5b0 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLEncoder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLEncoder.java@@ -16,60 +16,50 @@*/package org.apache.lucene.search.highlight;-/**- * Simple {@link Encoder} implementation to escape text for HTML output- *- */+/** Simple {@link Encoder} implementation to escape text for HTML output */public class SimpleHTMLEncoder implements Encoder {- public SimpleHTMLEncoder() {- }+ public SimpleHTMLEncoder() {}@Override- public String encodeText(String originalText)- {+ public String encodeText(String originalText) {return htmlEncode(originalText);}- /**- * Encode string into HTML- */- public final static String htmlEncode(String plainText)- {- if (plainText == null || plainText.length() == 0)- {+ /** Encode string into HTML */+ public static final String htmlEncode(String plainText) {+ if (plainText == null || plainText.length() == 0) {return "";}StringBuilder result = new StringBuilder(plainText.length());- for (int index=0; index<plainText.length(); index++)- {+ for (int index = 0; index < plainText.length(); index++) {char ch = plainText.charAt(index);switch (ch) {- case '"':- result.append(""");- break;- case '&':- result.append("&");- break;- case '<':- result.append("<");- break;- case '>':- result.append(">");- break;- case '\'':- result.append("'");- break;- case '/':- result.append("/");- break;- default:- result.append(ch);+ case '"':+ result.append(""");+ break;+ case '&':+ result.append("&");+ break;+ case '<':+ result.append("<");+ break;+ case '>':+ result.append(">");+ break;+ case '\'':+ result.append("'");+ break;+ case '/':+ result.append("/");+ break;+ default:+ result.append(ch);}}return result.toString();}-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.javaindex fec39711908..b129890fb97 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleHTMLFormatter.java@@ -16,15 +16,12 @@*/package org.apache.lucene.search.highlight;-/**- * Simple {@link Formatter} implementation to highlight terms with a pre and- * post tag.- */+/** Simple {@link Formatter} implementation to highlight terms with a pre and post tag. */public class SimpleHTMLFormatter implements Formatter {-+private static final String DEFAULT_PRE_TAG = "<B>";private static final String DEFAULT_POST_TAG = "</B>";-+private String preTag;private String postTag;@@ -49,11 +46,11 @@ public class SimpleHTMLFormatter implements Formatter {// Allocate StringBuilder with the right number of characters from the// beginning, to avoid char[] allocations in the middle of appends.- StringBuilder returnBuffer = new StringBuilder(preTag.length() + originalText.length() + postTag.length());+ StringBuilder returnBuffer =+ new StringBuilder(preTag.length() + originalText.length() + postTag.length());returnBuffer.append(preTag);returnBuffer.append(originalText);returnBuffer.append(postTag);return returnBuffer.toString();}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.javaindex 7b708d8dcf1..735f39cae77 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java@@ -16,19 +16,16 @@*/package org.apache.lucene.search.highlight;-import java.util.List;-import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.search.spans.Spans;-/**- * {@link Fragmenter} implementation which breaks text up into same-size- * fragments but does not split up {@link Spans}. This is a simple sample class.+ * {@link Fragmenter} implementation which breaks text up into same-size fragments but does not+ * split up {@link Spans}. This is a simple sample class.*/public class SimpleSpanFragmenter implements Fragmenter {private static final int DEFAULT_FRAGMENT_SIZE = 100;@@ -42,9 +39,7 @@ public class SimpleSpanFragmenter implements Fragmenter {private PositionIncrementAttribute posIncAtt;private OffsetAttribute offsetAtt;- /**- * @param queryScorer QueryScorer that was used to score hits- */+ /** @param queryScorer QueryScorer that was used to score hits */public SimpleSpanFragmenter(QueryScorer queryScorer) {this(queryScorer, DEFAULT_FRAGMENT_SIZE);}@@ -57,7 +52,7 @@ public class SimpleSpanFragmenter implements Fragmenter {this.fragmentSize = fragmentSize;this.queryScorer = queryScorer;}-+/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Fragmenter#isNewFragment()*/@@ -84,9 +79,10 @@ public class SimpleSpanFragmenter implements Fragmenter {}}- boolean isNewFrag = offsetAtt.endOffset() >= (fragmentSize * currentNumFrags)- && (textSize - offsetAtt.endOffset()) >= (fragmentSize >>> 1);-+ boolean isNewFrag =+ offsetAtt.endOffset() >= (fragmentSize * currentNumFrags)+ && (textSize - offsetAtt.endOffset()) >= (fragmentSize >>> 1);+if (isNewFrag) {currentNumFrags++;}@@ -94,7 +90,6 @@ public class SimpleSpanFragmenter implements Fragmenter {return isNewFrag;}-/* (non-Javadoc)* @see org.apache.lucene.search.highlight.Fragmenter#start(java.lang.String, org.apache.lucene.analysis.TokenStream)*/diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.javaindex e1088144128..0871f5e149f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/SpanGradientFormatter.java@@ -16,28 +16,25 @@*/package org.apache.lucene.search.highlight;/**- * Formats text with different color intensity depending on the score of the- * term using the span tag. GradientFormatter uses a bgcolor argument to the font tag which- * doesn't work in Mozilla, thus this class.+ * Formats text with different color intensity depending on the score of the term using the span+ * tag. GradientFormatter uses a bgcolor argument to the font tag which doesn't work in Mozilla,+ * thus this class.** @see GradientFormatter*/--public class SpanGradientFormatter- extends GradientFormatter {- public SpanGradientFormatter(float maxScore, String minForegroundColor,- String maxForegroundColor, String minBackgroundColor,- String maxBackgroundColor) {- super(maxScore, minForegroundColor,- maxForegroundColor, minBackgroundColor,- maxBackgroundColor);+public class SpanGradientFormatter extends GradientFormatter {+ public SpanGradientFormatter(+ float maxScore,+ String minForegroundColor,+ String maxForegroundColor,+ String minBackgroundColor,+ String maxBackgroundColor) {+ super(maxScore, minForegroundColor, maxForegroundColor, minBackgroundColor, maxBackgroundColor);}-@Overridepublic String highlightTerm(String originalText, TokenGroup tokenGroup) {- if (tokenGroup.getTotalScore() == 0)- return originalText;+ if (tokenGroup.getTotalScore() == 0) return originalText;float score = tokenGroup.getTotalScore();if (score == 0) {return originalText;@@ -63,7 +60,9 @@ public class SpanGradientFormatterreturn sb.toString();}- // guess how much extra text we'll add to the text we're highlighting to try to avoid a StringBuilder resize- private static final String TEMPLATE = "<span style=\"background: #EEEEEE; color: #000000;\">...</span>";+ // guess how much extra text we'll add to the text we're highlighting to try to avoid a+ // StringBuilder resize+ private static final String TEMPLATE =+ "<span style=\"background: #EEEEEE; color: #000000;\">...</span>";private static final int EXTRA = TEMPLATE.length();}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.javaindex 0b8cb906f77..0b782cf5137 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java@@ -19,7 +19,6 @@ package org.apache.lucene.search.highlight;import java.io.IOException;import java.util.Collections;import java.util.Iterator;-import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;@@ -50,25 +49,26 @@ public class TermVectorLeafReader extends LeafReader {private final FieldInfos fieldInfos;public TermVectorLeafReader(String field, Terms terms) {- fields = new Fields() {- @Override- public Iterator<String> iterator() {- return Collections.singletonList(field).iterator();- }-- @Override- public Terms terms(String fld) throws IOException {- if (!field.equals(fld)) {- return null;- }- return terms;- }-- @Override- public int size() {- return 1;- }- };+ fields =+ new Fields() {+ @Override+ public Iterator<String> iterator() {+ return Collections.singletonList(field).iterator();+ }++ @Override+ public Terms terms(String fld) throws IOException {+ if (!field.equals(fld)) {+ return null;+ }+ return terms;+ }++ @Override+ public int size() {+ return 1;+ }+ };IndexOptions indexOptions;if (!terms.hasFreqs()) {@@ -80,15 +80,28 @@ public class TermVectorLeafReader extends LeafReader {} else {indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;}- FieldInfo fieldInfo = new FieldInfo(field, 0,- true, true, terms.hasPayloads(),- indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, 0, VectorValues.SearchStrategy.NONE, false);- fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});- }-- @Override- protected void doClose() throws IOException {- }+ FieldInfo fieldInfo =+ new FieldInfo(+ field,+ 0,+ true,+ true,+ terms.hasPayloads(),+ indexOptions,+ DocValuesType.NONE,+ -1,+ Collections.emptyMap(),+ 0,+ 0,+ 0,+ 0,+ VectorValues.SearchStrategy.NONE,+ false);+ fieldInfos = new FieldInfos(new FieldInfo[] {fieldInfo});+ }++ @Override+ protected void doClose() throws IOException {}@Overridepublic Terms terms(String field) throws IOException {@@ -122,7 +135,7 @@ public class TermVectorLeafReader extends LeafReader {@Overridepublic NumericDocValues getNormValues(String field) throws IOException {- return null;//Is this needed? See MemoryIndex for a way to do it.+ return null; // Is this needed? See MemoryIndex for a way to do it.}@Override@@ -146,8 +159,7 @@ public class TermVectorLeafReader extends LeafReader {}@Override- public void checkIntegrity() throws IOException {- }+ public void checkIntegrity() throws IOException {}@Overridepublic Fields getTermVectors(int docID) throws IOException {@@ -168,8 +180,7 @@ public class TermVectorLeafReader extends LeafReader {}@Override- public void document(int docID, StoredFieldVisitor visitor) throws IOException {- }+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {}@Overridepublic LeafMetaData getMetaData() {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.javaindex d71954f50ea..3ae4e9f2716 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TextFragment.java@@ -15,56 +15,39 @@* limitations under the License.*/package org.apache.lucene.search.highlight;-/**- * Low-level class used to record information about a section of a document- * with a score.- *- *- */-public class TextFragment-{+/** Low-level class used to record information about a section of a document with a score. */+public class TextFragment {CharSequence markedUpText;int fragNum;int textStartPos;int textEndPos;float score;- public TextFragment(CharSequence markedUpText,int textStartPos, int fragNum)- {- this.markedUpText=markedUpText;+ public TextFragment(CharSequence markedUpText, int textStartPos, int fragNum) {+ this.markedUpText = markedUpText;this.textStartPos = textStartPos;this.fragNum = fragNum;}- void setScore(float score)- {- this.score=score;+ void setScore(float score) {+ this.score = score;}- public float getScore()- {++ public float getScore() {return score;}- /**- * @param frag2 Fragment to be merged into this one- */- public void merge(TextFragment frag2)- {+ /** @param frag2 Fragment to be merged into this one */+ public void merge(TextFragment frag2) {textEndPos = frag2.textEndPos;- score=Math.max(score,frag2.score);+ score = Math.max(score, frag2.score);}- /**- * @return true if this fragment follows the one passed- */- public boolean follows(TextFragment fragment)- {+ /** @return true if this fragment follows the one passed */+ public boolean follows(TextFragment fragment) {return textStartPos == fragment.textEndPos;}- /**- * @return the fragment sequence number- */- public int getFragNum()- {+ /** @return the fragment sequence number */+ public int getFragNum() {return fragNum;}@@ -74,5 +57,4 @@ public class TextFragmentpublic String toString() {return markedUpText.subSequence(textStartPos, textEndPos).toString();}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.javaindex ebb37d79956..eb903fc9beb 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java@@ -21,8 +21,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;/**- * One, or several overlapping tokens, along with the score(s) and the scope of- * the original text.+ * One, or several overlapping tokens, along with the score(s) and the scope of the original text.*/public class TokenGroup {@@ -82,7 +81,6 @@ public class TokenGroup {}/**- ** @param index a value between 0 and numTokens -1* @return the "n"th score*/@@ -91,33 +89,28 @@ public class TokenGroup {}/**- * @return the earliest start offset in the original text of a matching token in this group (score > 0), or- * if there are none then the earliest offset of any token in the group.+ * @return the earliest start offset in the original text of a matching token in this group (score+ * > 0), or if there are none then the earliest offset of any token in the group.*/public int getStartOffset() {return matchStartOffset;}/**- * @return the latest end offset in the original text of a matching token in this group (score > 0), or- * if there are none then {@link #getEndOffset()}.+ * @return the latest end offset in the original text of a matching token in this group (score+ * > 0), or if there are none then {@link #getEndOffset()}.*/public int getEndOffset() {return matchEndOffset;}- /**- * @return the number of tokens in this group- */+ /** @return the number of tokens in this group */public int getNumTokens() {return numTokens;}- /**- * @return all tokens' scores summed up- */+ /** @return all tokens' scores summed up */public float getTotalScore() {return tot;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.javaindex 0c0a63f9400..a3d8dbb3c35 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java@@ -1,6 +1,3 @@-/*- * Created on 28-Oct-2004- *//** Licensed to the Apache Software Foundation (ASF) under one or more* contributor license agreements. See the NOTICE file distributed with@@ -20,7 +17,6 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.document.Document;@@ -29,8 +25,9 @@ import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Terms;/**- * Convenience methods for obtaining a {@link TokenStream} for use with the {@link Highlighter} - can obtain from- * term vectors with offsets and positions or from an Analyzer re-parsing the stored content.+ * Convenience methods for obtaining a {@link TokenStream} for use with the {@link Highlighter} -+ * can obtain from term vectors with offsets and positions or from an Analyzer re-parsing the stored+ * content.** @see TokenStreamFromTermVector*/@@ -39,23 +36,26 @@ public class TokenSources {private TokenSources() {}/**- * Get a token stream from either un-inverting a term vector if possible, or by analyzing the text.+ * Get a token stream from either un-inverting a term vector if possible, or by analyzing the+ * text.*- * WARNING: Don't call this if there is more than one value for this field. If there are, and if there are term- * vectors, then there is a single tokenstream with offsets suggesting all the field values were concatenated.+ * <p>WARNING: Don't call this if there is more than one value for this field. If there are, and+ * if there are term vectors, then there is a single tokenstream with offsets suggesting all the+ * field values were concatenated.** @param field The field to either get term vectors from or to analyze the text from.- * @param tvFields from {@link IndexReader#getTermVectors(int)}. Possibly null. For performance, this instance should- * be re-used for the same document (e.g. when highlighting multiple fields).+ * @param tvFields from {@link IndexReader#getTermVectors(int)}. Possibly null. For performance,+ * this instance should be re-used for the same document (e.g. when highlighting multiple+ * fields).* @param text the text to analyze, failing term vector un-inversion* @param analyzer the analyzer to analyze {@code text} with, failing term vector un-inversion- * @param maxStartOffset Terms with a startOffset greater than this aren't returned. Use -1 for no limit.- * Suggest using {@link Highlighter#getMaxDocCharsToAnalyze()} - 1.- *+ * @param maxStartOffset Terms with a startOffset greater than this aren't returned. Use -1 for no+ * limit. Suggest using {@link Highlighter#getMaxDocCharsToAnalyze()} - 1.* @return a token stream from either term vectors, or from analyzing the text. Never null.*/- public static TokenStream getTokenStream(String field, Fields tvFields, String text, Analyzer analyzer,- int maxStartOffset) throws IOException {+ public static TokenStream getTokenStream(+ String field, Fields tvFields, String text, Analyzer analyzer, int maxStartOffset)+ throws IOException {TokenStream tokenStream = getTermVectorTokenStreamOrNull(field, tvFields, maxStartOffset);if (tokenStream != null) {return tokenStream;@@ -68,19 +68,20 @@ public class TokenSources {}/**- * Get a token stream by un-inverting the term vector. This method returns null if {@code tvFields} is null- * or if the field has no term vector, or if the term vector doesn't have offsets. Positions are recommended on the- * term vector but it isn't strictly required.+ * Get a token stream by un-inverting the term vector. This method returns null if {@code+ * tvFields} is null or if the field has no term vector, or if the term vector doesn't have+ * offsets. Positions are recommended on the term vector but it isn't strictly required.** @param field The field to get term vectors from.- * @param tvFields from {@link IndexReader#getTermVectors(int)}. Possibly null. For performance, this instance should- * be re-used for the same document (e.g. when highlighting multiple fields).- * @param maxStartOffset Terms with a startOffset greater than this aren't returned. Use -1 for no limit.- * Suggest using {@link Highlighter#getMaxDocCharsToAnalyze()} - 1+ * @param tvFields from {@link IndexReader#getTermVectors(int)}. Possibly null. For performance,+ * this instance should be re-used for the same document (e.g. when highlighting multiple+ * fields).+ * @param maxStartOffset Terms with a startOffset greater than this aren't returned. Use -1 for no+ * limit. Suggest using {@link Highlighter#getMaxDocCharsToAnalyze()} - 1* @return a token stream from term vectors. Null if no term vectors with the right options.*/- public static TokenStream getTermVectorTokenStreamOrNull(String field, Fields tvFields, int maxStartOffset)- throws IOException {+ public static TokenStream getTermVectorTokenStreamOrNull(+ String field, Fields tvFields, int maxStartOffset) throws IOException {if (tvFields == null) {return null;}@@ -93,26 +94,24 @@ public class TokenSources {/*** A convenience method that tries to first get a {@link TokenStreamFromTermVector} for the- * specified docId, then, falls back to using the passed in- * {@link org.apache.lucene.document.Document} to retrieve the TokenStream.- * This is useful when you already have the document, but would prefer to use- * the vector first.+ * specified docId, then, falls back to using the passed in {@link+ * org.apache.lucene.document.Document} to retrieve the TokenStream. This is useful when you+ * already have the document, but would prefer to use the vector first.*- * @param reader The {@link org.apache.lucene.index.IndexReader} to use to try- * and get the vector from+ * @param reader The {@link org.apache.lucene.index.IndexReader} to use to try and get the vector+ * from* @param docId The docId to retrieve.* @param field The field to retrieve on the document* @param document The document to fall back on- * @param analyzer The analyzer to use for creating the TokenStream if the- * vector doesn't exist- * @return The {@link org.apache.lucene.analysis.TokenStream} for the- * {@link org.apache.lucene.index.IndexableField} on the- * {@link org.apache.lucene.document.Document}+ * @param analyzer The analyzer to use for creating the TokenStream if the vector doesn't exist+ * @return The {@link org.apache.lucene.analysis.TokenStream} for the {@link+ * org.apache.lucene.index.IndexableField} on the {@link org.apache.lucene.document.Document}* @throws IOException if there was an error loading*/@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getAnyTokenStream(IndexReader reader, int docId,- String field, Document document, Analyzer analyzer) throws IOException {+ public static TokenStream getAnyTokenStream(+ IndexReader reader, int docId, String field, Document document, Analyzer analyzer)+ throws IOException {TokenStream ts = null;Fields vectors = reader.getTermVectors(docId);@@ -131,17 +130,16 @@ public class TokenSources {}/**- * A convenience method that tries a number of approaches to getting a token- * stream. The cost of finding there are no termVectors in the index is- * minimal (1000 invocations still registers 0 ms). So this "lazy" (flexible?)- * approach to coding is probably acceptable- *+ * A convenience method that tries a number of approaches to getting a token stream. The cost of+ * finding there are no termVectors in the index is minimal (1000 invocations still registers 0+ * ms). So this "lazy" (flexible?) approach to coding is probably acceptable+ ** @return null if field not stored correctly* @throws IOException If there is a low-level I/O error*/@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getAnyTokenStream(IndexReader reader, int docId,- String field, Analyzer analyzer) throws IOException {+ public static TokenStream getAnyTokenStream(+ IndexReader reader, int docId, String field, Analyzer analyzer) throws IOException {TokenStream ts = null;Fields vectors = reader.getTermVectors(docId);@@ -161,18 +159,18 @@ public class TokenSources {/** Simply calls {@link #getTokenStream(org.apache.lucene.index.Terms)} now. */@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getTokenStream(Terms vector,- boolean tokenPositionsGuaranteedContiguous) throws IOException {+ public static TokenStream getTokenStream(Terms vector, boolean tokenPositionsGuaranteedContiguous)+ throws IOException {return getTokenStream(vector);}/**- * Returns a token stream generated from a {@link Terms}. This- * can be used to feed the highlighter with a pre-parsed token- * stream. The {@link Terms} must have offsets available. If there are no positions available,- * all tokens will have position increments reflecting adjacent tokens, or coincident when terms- * share a start offset. If there are stopwords filtered from the index, you probably want to ensure- * term vectors have positions so that phrase queries won't match across stopwords.+ * Returns a token stream generated from a {@link Terms}. This can be used to feed the highlighter+ * with a pre-parsed token stream. The {@link Terms} must have offsets available. If there are no+ * positions available, all tokens will have position increments reflecting adjacent tokens, or+ * coincident when terms share a start offset. If there are stopwords filtered from the index, you+ * probably want to ensure term vectors have positions so that phrase queries won't match across+ * stopwords.** @throws IllegalArgumentException if no offsets are available*/@@ -181,7 +179,7 @@ public class TokenSources {if (!tpv.hasOffsets()) {throw new IllegalArgumentException("Highlighting requires offsets from the TokenStream.");- //TokenStreamFromTermVector can handle a lack of offsets if there are positions. But+ // TokenStreamFromTermVector can handle a lack of offsets if there are positions. But// highlighters require offsets, so we insist here.}@@ -189,22 +187,21 @@ public class TokenSources {}/**- * Returns a {@link TokenStream} with positions and offsets constructed from- * field termvectors. If the field has no termvectors or offsets- * are not included in the termvector, return null. See {@link #getTokenStream(org.apache.lucene.index.Terms)}- * for an explanation of what happens when positions aren't present.+ * Returns a {@link TokenStream} with positions and offsets constructed from field termvectors. If+ * the field has no termvectors or offsets are not included in the termvector, return null. See+ * {@link #getTokenStream(org.apache.lucene.index.Terms)} for an explanation of what happens when+ * positions aren't present.** @param reader the {@link IndexReader} to retrieve term vectors from* @param docId the document to retrieve termvectors for* @param field the field to retrieve termvectors for* @return a {@link TokenStream}, or null if offsets are not available* @throws IOException If there is a low-level I/O error- ** @see #getTokenStream(org.apache.lucene.index.Terms)*/@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getTokenStreamWithOffsets(IndexReader reader, int docId,- String field) throws IOException {+ public static TokenStream getTokenStreamWithOffsets(IndexReader reader, int docId, String field)+ throws IOException {Fields vectors = reader.getTermVectors(docId);if (vectors == null) {@@ -219,32 +216,29 @@ public class TokenSources {if (!vector.hasOffsets()) {return null;}-+return getTokenStream(vector);}@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getTokenStream(IndexReader reader, int docId,- String field, Analyzer analyzer) throws IOException {+ public static TokenStream getTokenStream(+ IndexReader reader, int docId, String field, Analyzer analyzer) throws IOException {Document doc = reader.document(docId);return getTokenStream(doc, field, analyzer);}@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getTokenStream(Document doc, String field,- Analyzer analyzer) {+ public static TokenStream getTokenStream(Document doc, String field, Analyzer analyzer) {String contents = doc.get(field);if (contents == null) {- throw new IllegalArgumentException("Field " + field- + " in document is not stored and cannot be analyzed");+ throw new IllegalArgumentException(+ "Field " + field + " in document is not stored and cannot be analyzed");}return getTokenStream(field, contents, analyzer);}@Deprecated // maintenance reasons LUCENE-6445- public static TokenStream getTokenStream(String field, String contents,- Analyzer analyzer) {+ public static TokenStream getTokenStream(String field, String contents, Analyzer analyzer) {return analyzer.tokenStream(field, contents);}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.javaindex 6353930abb5..7d4932c4355 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java@@ -17,7 +17,6 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;@@ -34,17 +33,19 @@ import org.apache.lucene.util.Counter;import org.apache.lucene.util.UnicodeUtil;/**- * TokenStream created from a term vector field. The term vector requires positions and/or offsets (either). If you- * want payloads add PayloadAttributeImpl (as you would normally) but don't assume the attribute is already added just- * because you know the term vector has payloads, since the first call to incrementToken() will observe if you asked- * for them and if not then won't get them. This TokenStream supports an efficient {@link #reset()}, so there's- * no need to wrap with a caching impl.- * <p>- * The implementation will create an array of tokens indexed by token position. As long as there aren't massive jumps- * in positions, this is fine. And it assumes there aren't large numbers of tokens at the same position, since it adds- * them to a linked-list per position in O(N^2) complexity. When there aren't positions in the term vector, it divides- * the startOffset by 8 to use as a temporary substitute. In that case, tokens with the same startOffset will occupy- * the same final position; otherwise tokens become adjacent.+ * TokenStream created from a term vector field. The term vector requires positions and/or offsets+ * (either). If you want payloads add PayloadAttributeImpl (as you would normally) but don't assume+ * the attribute is already added just because you know the term vector has payloads, since the+ * first call to incrementToken() will observe if you asked for them and if not then won't get them.+ * This TokenStream supports an efficient {@link #reset()}, so there's no need to wrap with a+ * caching impl.+ *+ * <p>The implementation will create an array of tokens indexed by token position. As long as there+ * aren't massive jumps in positions, this is fine. And it assumes there aren't large numbers of+ * tokens at the same position, since it adds them to a linked-list per position in O(N^2)+ * complexity. When there aren't positions in the term vector, it divides the startOffset by 8 to+ * use as a temporary substitute. In that case, tokens with the same startOffset will occupy the+ * same final position; otherwise tokens become adjacent.** @lucene.internal*/@@ -58,28 +59,29 @@ public final class TokenStreamFromTermVector extends TokenStream {private final int maxStartOffset;- private OffsetAttribute offsetAttribute;//maybe null+ private OffsetAttribute offsetAttribute; // maybe null- private PayloadAttribute payloadAttribute;//maybe null+ private PayloadAttribute payloadAttribute; // maybe null- private CharsRefBuilder termCharsBuilder;//term data here+ private CharsRefBuilder termCharsBuilder; // term data here- private BytesRefArray payloadsBytesRefArray;//only used when payloadAttribute is non-null- private BytesRefBuilder spareBytesRefBuilder;//only used when payloadAttribute is non-null+ private BytesRefArray payloadsBytesRefArray; // only used when payloadAttribute is non-null+ private BytesRefBuilder spareBytesRefBuilder; // only used when payloadAttribute is non-nullprivate TokenLL firstToken = null; // the head of a linked-listprivate TokenLL incrementToken = null;- private boolean initialized = false;//lazy+ private boolean initialized = false; // lazy/**- * Constructor. The uninversion doesn't happen here; it's delayed till the first call to- * {@link #incrementToken}.+ * Constructor. The uninversion doesn't happen here; it's delayed till the first call to {@link+ * #incrementToken}.*- * @param vector Terms that contains the data for- * creating the TokenStream. Must have positions and/or offsets.- * @param maxStartOffset if a token's start offset exceeds this then the token is not added. -1 disables the limit.+ * @param vector Terms that contains the data for creating the TokenStream. Must have positions+ * and/or offsets.+ * @param maxStartOffset if a token's start offset exceeds this then the token is not added. -1+ * disables the limit.*/public TokenStreamFromTermVector(Terms vector, int maxStartOffset) throws IOException {this.maxStartOffset = maxStartOffset < 0 ? Integer.MAX_VALUE : maxStartOffset;@@ -93,7 +95,9 @@ public final class TokenStreamFromTermVector extends TokenStream {positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);}- public Terms getTermVectorTerms() { return vector; }+ public Terms getTermVectorTerms() {+ return vector;+ }@Overridepublic void reset() throws IOException {@@ -101,7 +105,8 @@ public final class TokenStreamFromTermVector extends TokenStream {super.reset();}- //We delay initialization because we can see which attributes the consumer wants, particularly payloads+ // We delay initialization because we can see which attributes the consumer wants, particularly+ // payloadsprivate void init() throws IOException {assert !initialized;short dpEnumFlags = PostingsEnum.POSITIONS;@@ -110,7 +115,7 @@ public final class TokenStreamFromTermVector extends TokenStream {offsetAttribute = addAttribute(OffsetAttribute.class);}if (vector.hasPayloads() && hasAttribute(PayloadAttribute.class)) {- dpEnumFlags |= (PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS);//must ask for offsets too+ dpEnumFlags |= (PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS); // must ask for offsets toopayloadAttribute = getAttribute(PayloadAttribute.class);payloadsBytesRefArray = new BytesRefArray(Counter.newCounter());spareBytesRefBuilder = new BytesRefBuilder();@@ -118,7 +123,7 @@ public final class TokenStreamFromTermVector extends TokenStream {// We put term data heretermCharsBuilder = new CharsRefBuilder();- termCharsBuilder.grow((int) (vector.size() * 7));//7 is over-estimate of average term len+ termCharsBuilder.grow((int) (vector.size() * 7)); // 7 is over-estimate of average term len// Step 1: iterate termsEnum and create a token, placing into an array of tokens by position@@ -129,11 +134,12 @@ public final class TokenStreamFromTermVector extends TokenStream {final TermsEnum termsEnum = vector.iterator();BytesRef termBytesRef;PostingsEnum dpEnum = null;- CharsRefBuilder tempCharsRefBuilder = new CharsRefBuilder();//only for UTF8->UTF16 call- //int sumFreq = 0;+ CharsRefBuilder tempCharsRefBuilder = new CharsRefBuilder(); // only for UTF8->UTF16 call+ // int sumFreq = 0;while ((termBytesRef = termsEnum.next()) != null) {- //Grab the term (in same way as BytesRef.utf8ToString() but we don't want a String obj)- // note: if term vectors supported seek by ord then we might just keep an int and seek by ord on-demand+ // Grab the term (in same way as BytesRef.utf8ToString() but we don't want a String obj)+ // note: if term vectors supported seek by ord then we might just keep an int and seek by ord+ // on-demandtempCharsRefBuilder.grow(termBytesRef.length);final int termCharsLen = UnicodeUtil.UTF8toUTF16(termBytesRef, tempCharsRefBuilder.chars());final int termCharsOff = termCharsBuilder.length();@@ -143,7 +149,7 @@ public final class TokenStreamFromTermVector extends TokenStream {assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlierdpEnum.nextDoc();final int freq = dpEnum.freq();- //sumFreq += freq;+ // sumFreq += freq;for (int j = 0; j < freq; j++) {int pos = dpEnum.nextPosition();TokenLL token = new TokenLL();@@ -152,11 +158,12 @@ public final class TokenStreamFromTermVector extends TokenStream {if (offsetAttribute != null) {token.startOffset = dpEnum.startOffset();if (token.startOffset > maxStartOffset) {- continue;//filter this token out; exceeds threshold+ continue; // filter this token out; exceeds threshold}- token.endOffsetInc = (short) Math.min(dpEnum.endOffset() - token.startOffset, Short.MAX_VALUE);+ token.endOffsetInc =+ (short) Math.min(dpEnum.endOffset() - token.startOffset, Short.MAX_VALUE);if (pos == -1) {- pos = token.startOffset >> 3;//divide by 8+ pos = token.startOffset >> 3; // divide by 8}}@@ -165,10 +172,10 @@ public final class TokenStreamFromTermVector extends TokenStream {token.payloadIndex = payload == null ? -1 : payloadsBytesRefArray.append(payload);}- //Add token to an array indexed by position+ // Add token to an array indexed by positionif (positionedTokens.length <= pos) {- //grow, but not 2x since we think our original length estimate is close- TokenLL[] newPositionedTokens = new TokenLL[(int)((pos + 1) * 1.5f)];+ // grow, but not 2x since we think our original length estimate is close+ TokenLL[] newPositionedTokens = new TokenLL[(int) ((pos + 1) * 1.5f)];System.arraycopy(positionedTokens, 0, newPositionedTokens, 0, lastPosition + 1);positionedTokens = newPositionedTokens;}@@ -178,10 +185,12 @@ public final class TokenStreamFromTermVector extends TokenStream {}}-// System.out.println(String.format(-// "SumFreq: %5d Size: %4d SumFreq/size: %3.3f MaxPos: %4d MaxPos/SumFreq: %3.3f WastePct: %3.3f",-// sumFreq, vector.size(), (sumFreq / (float)vector.size()), lastPosition, ((float)lastPosition)/sumFreq,-// (originalPositionEstimate/(lastPosition + 1.0f))));+ // System.out.println(String.format(+ // "SumFreq: %5d Size: %4d SumFreq/size: %3.3f MaxPos: %4d MaxPos/SumFreq: %3.3f+ // WastePct: %3.3f",+ // sumFreq, vector.size(), (sumFreq / (float)vector.size()), lastPosition,+ // ((float)lastPosition)/sumFreq,+ // (originalPositionEstimate/(lastPosition + 1.0f))));// Step 2: Link all Tokens into a linked-list and set position increments as we go@@ -192,15 +201,15 @@ public final class TokenStreamFromTermVector extends TokenStream {if (token == null) {continue;}- //link+ // linkif (prevToken != null) {assert prevToken.next == null;- prevToken.next = token; //concatenate linked-list+ prevToken.next = token; // concatenate linked-list} else {assert firstToken == null;firstToken = token;}- //set increments+ // set incrementsif (vector.hasPositions()) {token.positionIncrement = pos - prevTokenPos;while (token.next != null) {@@ -227,18 +236,21 @@ public final class TokenStreamFromTermVector extends TokenStream {}private TokenLL[] initTokensArray() throws IOException {- // Estimate the number of position slots we need from term stats. We use some estimation factors taken from+ // Estimate the number of position slots we need from term stats. We use some estimation+ // factors taken from// Wikipedia that reduce the likelihood of needing to expand the array.int sumTotalTermFreq = (int) vector.getSumTotalTermFreq();assert sumTotalTermFreq != -1;- final int originalPositionEstimate = (int) (sumTotalTermFreq * 1.5);//less than 1 in 10 docs exceed this+ final int originalPositionEstimate =+ (int) (sumTotalTermFreq * 1.5); // less than 1 in 10 docs exceed this// This estimate is based on maxStartOffset. Err on the side of this being larger than needed.final int offsetLimitPositionEstimate = (int) (maxStartOffset / 5.0);// Take the smaller of the two estimates, but no smaller than 64- return new TokenLL[Math.max(64, Math.min(originalPositionEstimate, offsetLimitPositionEstimate))];+ return new TokenLL+ [Math.max(64, Math.min(originalPositionEstimate, offsetLimitPositionEstimate))];}@Override@@ -258,16 +270,19 @@ public final class TokenStreamFromTermVector extends TokenStream {return false;}clearAttributes();- termAttribute.copyBuffer(termCharsBuilder.chars(), incrementToken.termCharsOff, incrementToken.termCharsLen);+ termAttribute.copyBuffer(+ termCharsBuilder.chars(), incrementToken.termCharsOff, incrementToken.termCharsLen);positionIncrementAttribute.setPositionIncrement(incrementToken.positionIncrement);if (offsetAttribute != null) {- offsetAttribute.setOffset(incrementToken.startOffset, incrementToken.startOffset + incrementToken.endOffsetInc);+ offsetAttribute.setOffset(+ incrementToken.startOffset, incrementToken.startOffset + incrementToken.endOffsetInc);}if (payloadAttribute != null) {if (incrementToken.payloadIndex == -1) {payloadAttribute.setPayload(null);} else {- payloadAttribute.setPayload(payloadsBytesRefArray.get(spareBytesRefBuilder, incrementToken.payloadIndex));+ payloadAttribute.setPayload(+ payloadsBytesRefArray.get(spareBytesRefBuilder, incrementToken.payloadIndex));}}return true;@@ -286,9 +301,10 @@ public final class TokenStreamFromTermVector extends TokenStream {TokenLL next;- /** Given the head of a linked-list (possibly null) this inserts the token at the correct- * spot to maintain the desired order, and returns the head (which could be this token if it's the smallest).- * O(N^2) complexity but N should be a handful at most.+ /**+ * Given the head of a linked-list (possibly null) this inserts the token at the correct spot to+ * maintain the desired order, and returns the head (which could be this token if it's the+ * smallest). O(N^2) complexity but N should be a handful at most.*/TokenLL insertIntoSortedLinkedList(final TokenLL head) {assert next == null;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.javaindex ee9cf3a8ec8..ab273181ab5 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java@@ -16,17 +16,12 @@*/package org.apache.lucene.search.highlight;-import java.util.ArrayList;import java.util.Iterator;import java.util.List;--/**- * Lightweight class to hold term, weight, and positions used for scoring this- * term.- */-public class WeightedSpanTerm extends WeightedTerm{+/** Lightweight class to hold term, weight, and positions used for scoring this term. */+public class WeightedSpanTerm extends WeightedTerm {boolean positionSensitive;private List<PositionSpan> positionSpans = new ArrayList<>();@@ -43,8 +38,7 @@ public class WeightedSpanTerm extends WeightedTerm{/*** Checks to see if this term is valid at <code>position</code>.*- * @param position- * to check against valid term positions+ * @param position to check against valid term positions* @return true iff this term is a hit at this position*/public boolean checkPosition(int position) {@@ -80,7 +74,4 @@ public class WeightedSpanTerm extends WeightedTerm{public List<PositionSpan> getPositionSpans() {return positionSpans;}-}--diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.javaindex ee0a1875a98..9f75d657fbe 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java@@ -24,7 +24,6 @@ import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Set;-import org.apache.lucene.analysis.CachingTokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.index.BinaryDocValues;@@ -68,12 +67,13 @@ import org.apache.lucene.util.Bits;import org.apache.lucene.util.IOUtils;/**- * Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether- * {@link Term}s from the {@link Query} are contained in a supplied {@link TokenStream}.+ * Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether {@link+ * Term}s from the {@link Query} are contained in a supplied {@link TokenStream}.+ *+ * <p>In order to support additional, by default unsupported queries, subclasses can override {@link+ * #extract(Query, float, Map)} for extracting wrapped or delegate queries and {@link+ * #extractUnknownQuery(Query, Map)} to process custom leaf queries:*- * In order to support additional, by default unsupported queries, subclasses can override- * {@link #extract(Query, float, Map)} for extracting wrapped or delegate queries and- * {@link #extractUnknownQuery(Query, Map)} to process custom leaf queries:* <pre>* <code>* WeightedSpanTermExtractor extractor = new WeightedSpanTermExtractor() {@@ -99,7 +99,7 @@ import org.apache.lucene.util.IOUtils;public class WeightedSpanTermExtractor {private String fieldName;- private TokenStream tokenStream;//set subsequent to getWeightedSpanTerms* methods+ private TokenStream tokenStream; // set subsequent to getWeightedSpanTerms* methodsprivate String defaultField;private boolean expandMultiTermQuery;private boolean cachedTokenStream;@@ -117,15 +117,15 @@ public class WeightedSpanTermExtractor {}/**- * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied <code>Query</code>.- *- * @param query- * Query to extract Terms from- * @param terms- * Map to place created WeightedSpanTerms in+ * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied+ * <code>Query</code>.+ *+ * @param query Query to extract Terms from+ * @param terms Map to place created WeightedSpanTerms in* @throws IOException If there is a low-level I/O error*/- protected void extract(Query query, float boost, Map<String,WeightedSpanTerm> terms) throws IOException {+ protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms)+ throws IOException {if (query instanceof BoostQuery) {BoostQuery boostQuery = (BoostQuery) query;extract(boostQuery.getQuery(), boost * boostQuery.getBoost(), terms);@@ -151,13 +151,15 @@ public class WeightedSpanTermExtractor {int[] positions = phraseQuery.getPositions();if (positions.length >= 2) {// positions are in increasing order. max(0,...) is just a safeguard.- positionGaps = Math.max(0, positions[positions.length - 1] - positions[0] - positions.length + 1);+ positionGaps =+ Math.max(0, positions[positions.length - 1] - positions[0] - positions.length + 1);}- //if original slop is 0 then require inOrder+ // if original slop is 0 then require inOrderboolean inorder = (phraseQuery.getSlop() == 0);- SpanNearQuery sp = new SpanNearQuery(clauses, phraseQuery.getSlop() + positionGaps, inorder);+ SpanNearQuery sp =+ new SpanNearQuery(clauses, phraseQuery.getSlop() + positionGaps, inorder);extractWeightedSpanTerms(terms, sp, boost);}} else if (query instanceof TermQuery || query instanceof SynonymQuery) {@@ -170,7 +172,7 @@ public class WeightedSpanTermExtractor {extract(q, boost, terms);}} else if (query instanceof CommonTermsQuery) {- // specialized since rewriting would change the result query+ // specialized since rewriting would change the result query// this query is index sensitive.extractWeightedTerms(terms, query, boost);} else if (query instanceof DisjunctionMaxQuery) {@@ -190,7 +192,7 @@ public class WeightedSpanTermExtractor {}}- @SuppressWarnings({"unchecked","rawtypes"})+ @SuppressWarnings({"unchecked", "rawtypes"})final List<SpanQuery>[] disjunctLists = new List[maxPosition + 1];int distinctPositions = 0;@@ -211,8 +213,8 @@ public class WeightedSpanTermExtractor {final SpanQuery[] clauses = new SpanQuery[distinctPositions];for (List<SpanQuery> disjuncts : disjunctLists) {if (disjuncts != null) {- clauses[position++] = new SpanOrQuery(disjuncts- .toArray(new SpanQuery[disjuncts.size()]));+ clauses[position++] =+ new SpanOrQuery(disjuncts.toArray(new SpanQuery[disjuncts.size()]));} else {++positionGaps;}@@ -229,14 +231,14 @@ public class WeightedSpanTermExtractor {}}} else if (query instanceof MatchAllDocsQuery) {- //nothing+ // nothing} else if (query instanceof FunctionScoreQuery) {extract(((FunctionScoreQuery) query).getWrappedQuery(), boost, terms);} else if (isQueryUnsupported(query.getClass())) {// nothing} else {- if (query instanceof MultiTermQuery &&- (!expandMultiTermQuery || !fieldNameComparator(((MultiTermQuery)query).getField()))) {+ if (query instanceof MultiTermQuery+ && (!expandMultiTermQuery || !fieldNameComparator(((MultiTermQuery) query).getField()))) {return;}Query origQuery = query;@@ -248,7 +250,8 @@ public class WeightedSpanTermExtractor {rewritten = origQuery.rewrite(reader);}if (rewritten != origQuery) {- // only rewrite once and then flatten again - the rewritten query could have a special treatment+ // only rewrite once and then flatten again - the rewritten query could have a special+ // treatment// if this method is overwritten in a subclass or above in the next recursionextract(rewritten, boost, terms);} else {@@ -269,22 +272,22 @@ public class WeightedSpanTermExtractor {return false;}- protected void extractUnknownQuery(Query query,- Map<String, WeightedSpanTerm> terms) throws IOException {-+ protected void extractUnknownQuery(Query query, Map<String, WeightedSpanTerm> terms)+ throws IOException {+// for sub-classing to extract custom queries}/**- * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied <code>SpanQuery</code>.- *- * @param terms- * Map to place created WeightedSpanTerms in- * @param spanQuery- * SpanQuery to extract Terms from+ * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied+ * <code>SpanQuery</code>.+ *+ * @param terms Map to place created WeightedSpanTerms in+ * @param spanQuery SpanQuery to extract Terms from* @throws IOException If there is a low-level I/O error*/- protected void extractWeightedSpanTerms(Map<String,WeightedSpanTerm> terms, SpanQuery spanQuery, float boost) throws IOException {+ protected void extractWeightedSpanTerms(+ Map<String, WeightedSpanTerm> terms, SpanQuery spanQuery, float boost) throws IOException {Set<String> fieldNames;if (fieldName == null) {@@ -298,9 +301,9 @@ public class WeightedSpanTermExtractor {if (defaultField != null) {fieldNames.add(defaultField);}-+Map<String, SpanQuery> queries = new HashMap<>();-+Set<Term> nonWeightedTerms = new HashSet<>();final boolean mustRewriteQuery = mustRewriteQuery(spanQuery);final IndexSearcher searcher = new IndexSearcher(getLeafContext());@@ -325,7 +328,8 @@ public class WeightedSpanTermExtractor {q = spanQuery;}LeafReaderContext context = getLeafContext();- SpanWeight w = (SpanWeight) searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1);+ SpanWeight w =+ (SpanWeight) searcher.createWeight(searcher.rewrite(q), ScoreMode.COMPLETE_NO_SCORES, 1);Bits acceptDocs = context.reader().getLiveDocs();final Spans spans = w.getSpans(context, SpanWeight.Postings.POSITIONS);if (spans == null) {@@ -348,7 +352,7 @@ public class WeightedSpanTermExtractor {return;}- for (final Term queryTerm : nonWeightedTerms) {+ for (final Term queryTerm : nonWeightedTerms) {if (fieldNameComparator(queryTerm.field())) {WeightedSpanTerm weightedSpanTerm = terms.get(queryTerm.text());@@ -368,15 +372,15 @@ public class WeightedSpanTermExtractor {}/**- * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied <code>Query</code>.- *- * @param terms- * Map to place created WeightedSpanTerms in- * @param query- * Query to extract Terms from+ * Fills a <code>Map</code> with {@link WeightedSpanTerm}s using the terms from the supplied+ * <code>Query</code>.+ *+ * @param terms Map to place created WeightedSpanTerms in+ * @param query Query to extract Terms from* @throws IOException If there is a low-level I/O error*/- protected void extractWeightedTerms(Map<String,WeightedSpanTerm> terms, Query query, float boost) throws IOException {+ protected void extractWeightedTerms(Map<String, WeightedSpanTerm> terms, Query query, float boost)+ throws IOException {Set<Term> nonWeightedTerms = new HashSet<>();final IndexSearcher searcher = new IndexSearcher(getLeafContext());searcher.rewrite(query).visit(QueryVisitor.termCollector(nonWeightedTerms));@@ -390,12 +394,12 @@ public class WeightedSpanTermExtractor {}}- /**- * Necessary to implement matches for queries against <code>defaultField</code>- */+ /** Necessary to implement matches for queries against <code>defaultField</code> */protected boolean fieldNameComparator(String fieldNameToCheck) {- boolean rv = fieldName == null || fieldName.equals(fieldNameToCheck)- || (defaultField != null && defaultField.equals(fieldNameToCheck));+ boolean rv =+ fieldName == null+ || fieldName.equals(fieldNameToCheck)+ || (defaultField != null && defaultField.equals(fieldNameToCheck));return rv;}@@ -408,20 +412,23 @@ public class WeightedSpanTermExtractor {cacheIt = false;Terms termVectorTerms = ((TokenStreamFromTermVector) tokenStream).getTermVectorTerms();if (termVectorTerms.hasPositions() && termVectorTerms.hasOffsets()) {- internalReader = new TermVectorLeafReader(DelegatingLeafReader.FIELD_NAME, termVectorTerms);+ internalReader =+ new TermVectorLeafReader(DelegatingLeafReader.FIELD_NAME, termVectorTerms);}}// Use MemoryIndex (index/invert this tokenStream now)if (internalReader == null) {- final MemoryIndex indexer = new MemoryIndex(true, usePayloads);//offsets and payloads+ final MemoryIndex indexer = new MemoryIndex(true, usePayloads); // offsets and payloadsif (cacheIt) {assert !cachedTokenStream;- tokenStream = new CachingTokenFilter(new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));+ tokenStream =+ new CachingTokenFilter(new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));cachedTokenStream = true;indexer.addField(DelegatingLeafReader.FIELD_NAME, tokenStream);} else {- indexer.addField(DelegatingLeafReader.FIELD_NAME,+ indexer.addField(+ DelegatingLeafReader.FIELD_NAME,new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));}final IndexSearcher searcher = indexer.createSearcher();@@ -429,13 +436,13 @@ public class WeightedSpanTermExtractor {internalReader = ((LeafReaderContext) searcher.getTopReaderContext()).reader();}- //Now wrap it so we always use a common field.+ // Now wrap it so we always use a common field.this.internalReader = new DelegatingLeafReader(internalReader);}return internalReader.getContext();}-+/** This reader will just delegate every call to a single field in the wrapped* LeafReader. This way we only need to build this field once rather than@@ -450,7 +457,7 @@ public class WeightedSpanTermExtractor {@Overridepublic FieldInfos getFieldInfos() {- throw new UnsupportedOperationException();//TODO merge them+ throw new UnsupportedOperationException(); // TODO merge them}@Override@@ -462,17 +469,17 @@ public class WeightedSpanTermExtractor {public NumericDocValues getNumericDocValues(String field) throws IOException {return super.getNumericDocValues(FIELD_NAME);}-+@Overridepublic BinaryDocValues getBinaryDocValues(String field) throws IOException {return super.getBinaryDocValues(FIELD_NAME);}-+@Overridepublic SortedDocValues getSortedDocValues(String field) throws IOException {return super.getSortedDocValues(FIELD_NAME);}-+@Overridepublic NumericDocValues getNormValues(String field) throws IOException {return super.getNormValues(FIELD_NAME);@@ -490,41 +497,38 @@ public class WeightedSpanTermExtractor {}/**- * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>.- *+ * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>+ * TokenStream</code>.+ ** <p>- *- * @param query- * that caused hit- * @param tokenStream- * of text to be highlighted+ *+ * @param query that caused hit+ * @param tokenStream of text to be highlighted* @return Map containing WeightedSpanTerms* @throws IOException If there is a low-level I/O error*/- public Map<String,WeightedSpanTerm> getWeightedSpanTerms(Query query, float boost, TokenStream tokenStream)- throws IOException {+ public Map<String, WeightedSpanTerm> getWeightedSpanTerms(+ Query query, float boost, TokenStream tokenStream) throws IOException {return getWeightedSpanTerms(query, boost, tokenStream, null);}/**- * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>.- *+ * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>+ * TokenStream</code>.+ ** <p>- *- * @param query- * that caused hit- * @param tokenStream- * of text to be highlighted- * @param fieldName- * restricts Term's used based on field name+ *+ * @param query that caused hit+ * @param tokenStream of text to be highlighted+ * @param fieldName restricts Term's used based on field name* @return Map containing WeightedSpanTerms* @throws IOException If there is a low-level I/O error*/- public Map<String,WeightedSpanTerm> getWeightedSpanTerms(Query query, float boost, TokenStream tokenStream,- String fieldName) throws IOException {+ public Map<String, WeightedSpanTerm> getWeightedSpanTerms(+ Query query, float boost, TokenStream tokenStream, String fieldName) throws IOException {this.fieldName = fieldName;- Map<String,WeightedSpanTerm> terms = new PositionCheckingMap<>();+ Map<String, WeightedSpanTerm> terms = new PositionCheckingMap<>();this.tokenStream = tokenStream;try {extract(query, boost, terms);@@ -536,24 +540,22 @@ public class WeightedSpanTermExtractor {}/**- * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>. Uses a supplied- * <code>IndexReader</code> to properly weight terms (for gradient highlighting).- *+ * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>+ * TokenStream</code>. Uses a supplied <code>IndexReader</code> to properly weight terms (for+ * gradient highlighting).+ ** <p>- *- * @param query- * that caused hit- * @param tokenStream- * of text to be highlighted- * @param fieldName- * restricts Term's used based on field name- * @param reader- * to use for scoring+ *+ * @param query that caused hit+ * @param tokenStream of text to be highlighted+ * @param fieldName restricts Term's used based on field name+ * @param reader to use for scoring* @return Map of WeightedSpanTerms with quasi tf/idf scores* @throws IOException If there is a low-level I/O error*/- public Map<String,WeightedSpanTerm> getWeightedSpanTermsWithScores(Query query, float boost, TokenStream tokenStream, String fieldName,- IndexReader reader) throws IOException {+ public Map<String, WeightedSpanTerm> getWeightedSpanTermsWithScores(+ Query query, float boost, TokenStream tokenStream, String fieldName, IndexReader reader)+ throws IOException {if (fieldName != null) {this.fieldName = fieldName;} else {@@ -561,7 +563,7 @@ public class WeightedSpanTermExtractor {}this.tokenStream = tokenStream;- Map<String,WeightedSpanTerm> terms = new PositionCheckingMap<>();+ Map<String, WeightedSpanTerm> terms = new PositionCheckingMap<>();extract(query, boost, terms);int totalNumDocs = reader.maxDoc();@@ -585,65 +587,66 @@ public class WeightedSpanTermExtractor {protected void collectSpanQueryFields(SpanQuery spanQuery, Set<String> fieldNames) {if (spanQuery instanceof FieldMaskingSpanQuery) {- collectSpanQueryFields(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery(), fieldNames);+ collectSpanQueryFields(((FieldMaskingSpanQuery) spanQuery).getMaskedQuery(), fieldNames);} else if (spanQuery instanceof SpanFirstQuery) {- collectSpanQueryFields(((SpanFirstQuery)spanQuery).getMatch(), fieldNames);+ collectSpanQueryFields(((SpanFirstQuery) spanQuery).getMatch(), fieldNames);} else if (spanQuery instanceof SpanNearQuery) {- for (final SpanQuery clause : ((SpanNearQuery)spanQuery).getClauses()) {+ for (final SpanQuery clause : ((SpanNearQuery) spanQuery).getClauses()) {collectSpanQueryFields(clause, fieldNames);}} else if (spanQuery instanceof SpanNotQuery) {- collectSpanQueryFields(((SpanNotQuery)spanQuery).getInclude(), fieldNames);+ collectSpanQueryFields(((SpanNotQuery) spanQuery).getInclude(), fieldNames);} else if (spanQuery instanceof SpanOrQuery) {- for (final SpanQuery clause : ((SpanOrQuery)spanQuery).getClauses()) {+ for (final SpanQuery clause : ((SpanOrQuery) spanQuery).getClauses()) {collectSpanQueryFields(clause, fieldNames);}} else {fieldNames.add(spanQuery.getField());}}-+protected boolean mustRewriteQuery(SpanQuery spanQuery) {if (!expandMultiTermQuery) {return false; // Will throw UnsupportedOperationException in case of a SpanRegexQuery.} else if (spanQuery instanceof FieldMaskingSpanQuery) {- return mustRewriteQuery(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery());+ return mustRewriteQuery(((FieldMaskingSpanQuery) spanQuery).getMaskedQuery());} else if (spanQuery instanceof SpanFirstQuery) {- return mustRewriteQuery(((SpanFirstQuery)spanQuery).getMatch());+ return mustRewriteQuery(((SpanFirstQuery) spanQuery).getMatch());} else if (spanQuery instanceof SpanNearQuery) {- for (final SpanQuery clause : ((SpanNearQuery)spanQuery).getClauses()) {+ for (final SpanQuery clause : ((SpanNearQuery) spanQuery).getClauses()) {if (mustRewriteQuery(clause)) {return true;}}- return false;+ return false;} else if (spanQuery instanceof SpanNotQuery) {- SpanNotQuery spanNotQuery = (SpanNotQuery)spanQuery;- return mustRewriteQuery(spanNotQuery.getInclude()) || mustRewriteQuery(spanNotQuery.getExclude());+ SpanNotQuery spanNotQuery = (SpanNotQuery) spanQuery;+ return mustRewriteQuery(spanNotQuery.getInclude())+ || mustRewriteQuery(spanNotQuery.getExclude());} else if (spanQuery instanceof SpanOrQuery) {- for (final SpanQuery clause : ((SpanOrQuery)spanQuery).getClauses()) {+ for (final SpanQuery clause : ((SpanOrQuery) spanQuery).getClauses()) {if (mustRewriteQuery(clause)) {return true;}}- return false;+ return false;} else if (spanQuery instanceof SpanTermQuery) {return false;} else {return true;}}-+/**- * This class makes sure that if both position sensitive and insensitive- * versions of the same term are added, the position insensitive one wins.+ * This class makes sure that if both position sensitive and insensitive versions of the same term+ * are added, the position insensitive one wins.*/@SuppressWarnings("serial")- protected static class PositionCheckingMap<K> extends HashMap<K,WeightedSpanTerm> {+ protected static class PositionCheckingMap<K> extends HashMap<K, WeightedSpanTerm> {@Override- public void putAll(Map<? extends K,? extends WeightedSpanTerm> m) {- for (Map.Entry<? extends K,? extends WeightedSpanTerm> entry : m.entrySet())+ public void putAll(Map<? extends K, ? extends WeightedSpanTerm> m) {+ for (Map.Entry<? extends K, ? extends WeightedSpanTerm> entry : m.entrySet())this.put(entry.getKey(), entry.getValue());}@@ -658,9 +661,8 @@ public class WeightedSpanTermExtractor {}return prev;}-}-+public boolean getExpandMultiTermQuery() {return expandMultiTermQuery;}@@ -681,28 +683,30 @@ public class WeightedSpanTermExtractor {return cachedTokenStream;}- /** Returns the tokenStream which may have been wrapped in a CachingTokenFilter.- * getWeightedSpanTerms* sets the tokenStream, so don't call this before. */+ /**+ * Returns the tokenStream which may have been wrapped in a CachingTokenFilter.+ * getWeightedSpanTerms* sets the tokenStream, so don't call this before.+ */public TokenStream getTokenStream() {assert tokenStream != null;return tokenStream;}-+/**- * By default, {@link TokenStream}s that are not of the type- * {@link CachingTokenFilter} are wrapped in a {@link CachingTokenFilter} to- * ensure an efficient reset - if you are already using a different caching- * {@link TokenStream} impl and you don't want it to be wrapped, set this to- * false. This setting is ignored when a term vector based TokenStream is supplied,- * since it can be reset efficiently.+ * By default, {@link TokenStream}s that are not of the type {@link CachingTokenFilter} are+ * wrapped in a {@link CachingTokenFilter} to ensure an efficient reset - if you are already using+ * a different caching {@link TokenStream} impl and you don't want it to be wrapped, set this to+ * false. This setting is ignored when a term vector based TokenStream is supplied, since it can+ * be reset efficiently.*/public void setWrapIfNotCachingTokenFilter(boolean wrap) {this.wrapToCaching = wrap;}- /** A threshold of number of characters to analyze. When a TokenStream based on- * term vectors with offsets and positions are supplied, this setting- * does not apply. */+ /**+ * A threshold of number of characters to analyze. When a TokenStream based on term vectors with+ * offsets and positions are supplied, this setting does not apply.+ */protected final void setMaxDocCharsToAnalyze(int maxDocCharsToAnalyze) {this.maxDocCharsToAnalyze = maxDocCharsToAnalyze;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.javaindex 7fba55e17ff..adaad114799 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedTerm.java@@ -15,49 +15,33 @@* limitations under the License.*/package org.apache.lucene.search.highlight;-/** Lightweight class to hold term and a weight value used for scoring this term- */-public class WeightedTerm-{+/** Lightweight class to hold term and a weight value used for scoring this term */+public class WeightedTerm {float weight; // multiplier- String term; //stemmed form- public WeightedTerm (float weight,String term)- {- this.weight=weight;- this.term=term;- }+ String term; // stemmed form+ public WeightedTerm(float weight, String term) {+ this.weight = weight;+ this.term = term;+ }- /**- * @return the term value (stemmed)- */- public String getTerm()- {+ /** @return the term value (stemmed) */+ public String getTerm() {return term;}- /**- * @return the weight associated with this term- */- public float getWeight()- {+ /** @return the weight associated with this term */+ public float getWeight() {return weight;}- /**- * @param term the term value (stemmed)- */- public void setTerm(String term)- {+ /** @param term the term value (stemmed) */+ public void setTerm(String term) {this.term = term;}- /**- * @param weight the weight associated with this term- */- public void setWeight(float weight)- {+ /** @param weight the weight associated with this term */+ public void setWeight(float weight) {this.weight = weight;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/package-info.javaindex ba6e21a2187..a9ab1cafcbd 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/package-info.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/package-info.java@@ -17,13 +17,12 @@/*** Highlighting search terms.- * <p>- * The highlight package contains classes to provide "keyword in context" features- * typically used to highlight search terms in the text of results pages.- * The Highlighter class is the central component and can be used to extract the- * most interesting sections of a piece of text and highlight them, with the help of- * Fragmenter, fragment Scorer, and Formatter classes.- *+ *+ * <p>The highlight package contains classes to provide "keyword in context" features typically used+ * to highlight search terms in the text of results pages. The Highlighter class is the central+ * component and can be used to extract the most interesting sections of a piece of text and+ * highlight them, with the help of Fragmenter, fragment Scorer, and Formatter classes.+ ** <h2>Example Usage</h2>** <pre class="prettyprint">@@ -31,9 +30,9 @@* IndexSearcher searcher = new IndexSearcher(directory);* QueryParser parser = new QueryParser("notv", analyzer);* Query query = parser.parse("million");- *+ ** TopDocs hits = searcher.search(query, 10);- *+ ** SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter();* Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query));* for (int i = 0; i < 10; i++) {@@ -59,37 +58,38 @@* System.out.println("-------------");* }* </pre>- *+ ** <h2>New features 06/02/2005</h2>- *- * This release adds options for encoding (thanks to Nicko Cadell).- * An "Encoder" implementation such as the new SimpleHTMLEncoder class can be passed to the highlighter to encode- * all those non-xhtml standard characters such as & into legal values. This simple class may not suffice for- * some languages - Commons Lang has an implementation that could be used: escapeHtml(String) in+ *+ * This release adds options for encoding (thanks to Nicko Cadell). An "Encoder" implementation such+ * as the new SimpleHTMLEncoder class can be passed to the highlighter to encode all those non-xhtml+ * standard characters such as & into legal values. This simple class may not suffice for some+ * languages - Commons Lang has an implementation that could be used: escapeHtml(String) in* http://svn.apache.org/viewcvs.cgi/jakarta/commons/proper/lang/trunk/src/java/org/apache/commons/lang/StringEscapeUtils.java?rev=137958&view=markup- *+ ** <h2>New features 22/12/2004</h2>- *+ ** This release adds some new capabilities:+ ** <ol>- * <li>Faster highlighting using Term vector support</li>- * <li>New formatting options to use color intensity to show informational value</li>- * <li>Options for better summarization by using term IDF scores to influence fragment selection</li>+ * <li>Faster highlighting using Term vector support+ * <li>New formatting options to use color intensity to show informational value+ * <li>Options for better summarization by using term IDF scores to influence fragment selection* </ol>- *- * <p>- * The highlighter takes a TokenStream as input. Until now these streams have typically been produced- * using an Analyzer but the new class TokenSources provides helper methods for obtaining TokenStreams from- * the new TermVector position support (see latest CVS version).</p>- *- * <p>The new class GradientFormatter can use a scale of colors to highlight terms according to their score.- * A subtle use of color can help emphasise the reasons for matching (useful when doing "MoreLikeThis" queries and- * you want to see what the basis of the similarities are).</p>- *- * <p>The QueryScorer class has constructors that use an IndexReader to derive the IDF (inverse document frequency)- * for each term in order to influence the score. This is useful for helping to extracting the most significant sections- * of a document and in supplying scores used by the new GradientFormatter to color significant words more strongly.- * The QueryScorer.getMaxTermWeight method is useful when passed to the GradientFormatter constructor to define the top score- * which is associated with the top color.</p>+ *+ * <p>The highlighter takes a TokenStream as input. Until now these streams have typically been+ * produced using an Analyzer but the new class TokenSources provides helper methods for obtaining+ * TokenStreams from the new TermVector position support (see latest CVS version).+ *+ * <p>The new class GradientFormatter can use a scale of colors to highlight terms according to+ * their score. A subtle use of color can help emphasise the reasons for matching (useful when doing+ * "MoreLikeThis" queries and you want to see what the basis of the similarities are).+ *+ * <p>The QueryScorer class has constructors that use an IndexReader to derive the IDF (inverse+ * document frequency) for each term in order to influence the score. This is useful for helping to+ * extracting the most significant sections of a document and in supplying scores used by the new+ * GradientFormatter to color significant words more strongly. The QueryScorer.getMaxTermWeight+ * method is useful when passed to the GradientFormatter constructor to define the top score which+ * is associated with the top color.*/package org.apache.lucene.search.highlight;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/BreakIteratorShrinkingAdjuster.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/BreakIteratorShrinkingAdjuster.javaindex 21166da76a0..744ecf2d7ae 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/BreakIteratorShrinkingAdjuster.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/BreakIteratorShrinkingAdjuster.java@@ -20,8 +20,8 @@ import java.text.BreakIterator;import java.util.Locale;/**- * A {@link PassageAdjuster} that adjusts the {@link Passage} range to- * word boundaries hinted by the given {@link BreakIterator}.+ * A {@link PassageAdjuster} that adjusts the {@link Passage} range to word boundaries hinted by the+ * given {@link BreakIterator}.*/public class BreakIteratorShrinkingAdjuster implements PassageAdjuster {private final BreakIterator bi;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/CharSequenceIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/CharSequenceIterator.javaindex 701717f4710..40cb8cba283 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/CharSequenceIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/CharSequenceIterator.java@@ -18,9 +18,7 @@ package org.apache.lucene.search.matchhighlight;import java.text.CharacterIterator;-/**- * A {@link CharacterIterator} over a {@link CharSequence}.- */+/** A {@link CharacterIterator} over a {@link CharSequence}. */final class CharSequenceIterator implements CharacterIterator {private final CharSequence text;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.javaindex ece66937e0e..d04318a3175 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java@@ -26,16 +26,17 @@ import java.util.function.BiPredicate;import java.util.function.Predicate;/**- * A factory of {@link org.apache.lucene.search.matchhighlight.MatchHighlighter.FieldValueHighlighter} classes- * that cover typical use cases (verbatim values, highlights, abbreviations).+ * A factory of {@link+ * org.apache.lucene.search.matchhighlight.MatchHighlighter.FieldValueHighlighter} classes that+ * cover typical use cases (verbatim values, highlights, abbreviations).** @see MatchHighlighter#appendFieldHighlighter*/public final class FieldValueHighlighters {- private FieldValueHighlighters() {- }+ private FieldValueHighlighters() {}- private static abstract class AbstractFieldValueHighlighter implements MatchHighlighter.FieldValueHighlighter {+ private abstract static class AbstractFieldValueHighlighter+ implements MatchHighlighter.FieldValueHighlighter {private final BiPredicate<String, Boolean> testPredicate;protected AbstractFieldValueHighlighter(BiPredicate<String, Boolean> testPredicate) {@@ -49,18 +50,24 @@ public final class FieldValueHighlighters {}/**- * Displays up to {@code maxLeadingCharacters} of the field's value, regardless of whether it contained- * highlights or not.+ * Displays up to {@code maxLeadingCharacters} of the field's value, regardless of whether it+ * contained highlights or not.*/- public static MatchHighlighter.FieldValueHighlighter maxLeadingCharacters(int maxLeadingCharacters, String ellipsis, Set<String> fields) {+ public static MatchHighlighter.FieldValueHighlighter maxLeadingCharacters(+ int maxLeadingCharacters, String ellipsis, Set<String> fields) {PassageSelector passageSelector = defaultPassageSelector();PassageFormatter passageFormatter = new PassageFormatter(ellipsis, "", "");return new AbstractFieldValueHighlighter((field, hasMatches) -> fields.contains(field)) {@Override- public List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {List<Passage> bestPassages =- passageSelector.pickBest(contiguousValue, Collections.emptyList(), maxLeadingCharacters, 1, valueRanges);+ passageSelector.pickBest(+ contiguousValue, Collections.emptyList(), maxLeadingCharacters, 1, valueRanges);return passageFormatter.format(contiguousValue, bestPassages, valueRanges);}@@ -72,13 +79,10 @@ public final class FieldValueHighlighters {};}- /**- * Default preconfigured {@link PassageSelector}.- */+ /** Default preconfigured {@link PassageSelector}. */public static PassageSelector defaultPassageSelector() {return new PassageSelector(- PassageSelector.DEFAULT_SCORER,- new BreakIteratorShrinkingAdjuster());+ PassageSelector.DEFAULT_SCORER, new BreakIteratorShrinkingAdjuster());}/**@@ -90,24 +94,29 @@ public final class FieldValueHighlighters {PassageFormatter passageFormatter,Predicate<String> matchFields) {PassageSelector passageSelector = defaultPassageSelector();- return new AbstractFieldValueHighlighter((field, hasMatches) -> matchFields.test(field) && hasMatches) {+ return new AbstractFieldValueHighlighter(+ (field, hasMatches) -> matchFields.test(field) && hasMatches) {@Override- public List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {assert matchOffsets != null;List<Passage> bestPassages =- passageSelector.pickBest(contiguousValue, matchOffsets, maxPassageWindow, maxPassages, valueRanges);+ passageSelector.pickBest(+ contiguousValue, matchOffsets, maxPassageWindow, maxPassages, valueRanges);return passageFormatter.format(contiguousValue, bestPassages, valueRanges);}};}- /**- * Always returns raw field values, no highlighting or value truncation is applied.- */- public static MatchHighlighter.FieldValueHighlighter verbatimValue(String field, String... moreFields) {+ /** Always returns raw field values, no highlighting or value truncation is applied. */+ public static MatchHighlighter.FieldValueHighlighter verbatimValue(+ String field, String... moreFields) {HashSet<String> matchFields = new HashSet<>(Arrays.asList(moreFields));matchFields.add(field);return new AbstractFieldValueHighlighter((fld, hasMatches) -> matchFields.contains(fld)) {@@ -117,21 +126,30 @@ public final class FieldValueHighlighters {}@Override- public List<String> format(String field, String[] values, String contiguousValue, List<OffsetRange> valueRanges,- List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {return Arrays.asList(values);}};}/**- * Matches all fields and omits their value in the output (so that no highlight or value is emitted).+ * Matches all fields and omits their value in the output (so that no highlight or value is+ * emitted).*/public static MatchHighlighter.FieldValueHighlighter skipRemaining() {return new AbstractFieldValueHighlighter((field, hasMatches) -> true) {@Override- public List<String> format(String field, String[] values, String contiguousValue, List<OffsetRange> valueRanges,- List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {return null;}};diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.javaindex 277a3243e92..378c557707c 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java@@ -16,17 +16,6 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.analysis.Analyzer;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.DocumentStoredFieldVisitor;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexableField;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.Query;-import org.apache.lucene.search.ScoreDoc;-import org.apache.lucene.search.TopDocs;-import java.io.IOException;import java.io.UncheckedIOException;import java.util.ArrayList;@@ -39,14 +28,24 @@ import java.util.Map;import java.util.Objects;import java.util.function.Predicate;import java.util.stream.Stream;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.DocumentStoredFieldVisitor;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.IndexableField;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.ScoreDoc;+import org.apache.lucene.search.TopDocs;/**- * An example highlighter that combines several lower-level highlighting- * utilities in this package into a fully featured, ready-to-use component.- * <p>- * Note that if you need to customize or tweak the details of highlighting,- * it is better to assemble your own highlighter using those low-level- * building blocks, rather than extend or modify this one.+ * An example highlighter that combines several lower-level highlighting utilities in this package+ * into a fully featured, ready-to-use component.+ *+ * <p>Note that if you need to customize or tweak the details of highlighting, it is better to+ * assemble your own highlighter using those low-level building blocks, rather than extend or modify+ * this one.*/public class MatchHighlighter {private final IndexSearcher searcher;@@ -57,10 +56,9 @@ public class MatchHighlighter {private final List<FieldValueHighlighter> fieldHighlighters = new ArrayList<>();/**- * Actual per-field highlighter. Field highlighters are probed whether they- * are applicable to a particular combination of (field, hasMatches) pair. If a highlighter- * declares it is applicable, its {@link #format} method is invoked and the result- * is returned as the field's value.+ * Actual per-field highlighter. Field highlighters are probed whether they are applicable to a+ * particular combination of (field, hasMatches) pair. If a highlighter declares it is applicable,+ * its {@link #format} method is invoked and the result is returned as the field's value.** @see FieldValueHighlighters*/@@ -73,24 +71,24 @@ public class MatchHighlighter {*/boolean isApplicable(String field, boolean hasMatches);- /**- * Do format field values appropriately.- */- List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<QueryOffsetRange> matchOffsets);+ /** Do format field values appropriately. */+ List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<QueryOffsetRange> matchOffsets);/**- * @return Returns a set of fields that must be fetched for each document, regardless- * of whether they had matches or not. This is useful to load and return certain fields- * that should always be included (identifiers, document titles, etc.).+ * @return Returns a set of fields that must be fetched for each document, regardless of whether+ * they had matches or not. This is useful to load and return certain fields that should+ * always be included (identifiers, document titles, etc.).*/default Collection<String> alwaysFetchedFields() {return Collections.emptyList();}- /**- * Returns a new field value highlighter that is a combination of this one and another one.- */+ /** Returns a new field value highlighter that is a combination of this one and another one. */default FieldValueHighlighter or(FieldValueHighlighter other) {FieldValueHighlighter first = this;FieldValueHighlighter second = other;@@ -102,15 +100,20 @@ public class MatchHighlighter {return new FieldValueHighlighter() {@Overridepublic boolean isApplicable(String field, boolean hasMatches) {- return first.isApplicable(field, hasMatches)- || second.isApplicable(field, hasMatches);+ return first.isApplicable(field, hasMatches) || second.isApplicable(field, hasMatches);}@Override- public List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<QueryOffsetRange> matchOffsets) {+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<QueryOffsetRange> matchOffsets) {FieldValueHighlighter delegate =- first.isApplicable(field, matchOffsets != null && !matchOffsets.isEmpty()) ? first : second;+ first.isApplicable(field, matchOffsets != null && !matchOffsets.isEmpty())+ ? first+ : second;return delegate.format(field, values, contiguousValue, valueRanges, matchOffsets);}@@ -123,8 +126,8 @@ public class MatchHighlighter {}/**- * Append a new highlighter to field highlighters chain. The order of field highlighters- * is important (first-matching wins).+ * Append a new highlighter to field highlighters chain. The order of field highlighters is+ * important (first-matching wins).*/public MatchHighlighter appendFieldHighlighter(FieldValueHighlighter highlighter) {fieldHighlighters.add(highlighter);@@ -132,18 +135,14 @@ public class MatchHighlighter {return this;}- /**- * Always fetch the given set of fields for all input documents.- */+ /** Always fetch the given set of fields for all input documents. */public void alwaysFetchFields(String... fields) {for (String fld : fields) {fieldsAlwaysReturned.add(Objects.requireNonNull(fld));}}- /**- * Single document's highlights.- */+ /** Single document's highlights. */public static class DocHighlights {public final int docId;public final Map<String, List<String>> fields = new LinkedHashMap<>();@@ -153,9 +152,7 @@ public class MatchHighlighter {}}- /**- * An {@link OffsetRange} of a match, together with the source query that caused it.- */+ /** An {@link OffsetRange} of a match, together with the source query that caused it. */public static class QueryOffsetRange extends OffsetRange {public final Query query;@@ -174,8 +171,7 @@ public class MatchHighlighter {final int docId;private final LeafReader leafReader;private final int leafDocId;- private final LinkedHashMap<String, List<QueryOffsetRange>> matchRanges- = new LinkedHashMap<>();+ private final LinkedHashMap<String, List<QueryOffsetRange>> matchRanges = new LinkedHashMap<>();DocHit(int docId, LeafReader leafReader, int leafDocId) {this.docId = docId;@@ -184,21 +180,25 @@ public class MatchHighlighter {}void addMatches(Query query, Map<String, List<OffsetRange>> hits) {- hits.forEach((field, offsets) -> {- List<QueryOffsetRange> target = matchRanges.computeIfAbsent(field, (fld) -> new ArrayList<>());- offsets.forEach(o -> target.add(new QueryOffsetRange(query, o.from, o.to)));- });+ hits.forEach(+ (field, offsets) -> {+ List<QueryOffsetRange> target =+ matchRanges.computeIfAbsent(field, (fld) -> new ArrayList<>());+ offsets.forEach(o -> target.add(new QueryOffsetRange(query, o.from, o.to)));+ });}Document document(Predicate<String> needsField) throws IOException {// Only load the fields that have a chance to be highlighted.- DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor() {- @Override- public Status needsField(FieldInfo fieldInfo) {- return (matchRanges.containsKey(fieldInfo.name) ||- needsField.test(fieldInfo.name)) ? Status.YES : Status.NO;- }- };+ DocumentStoredFieldVisitor visitor =+ new DocumentStoredFieldVisitor() {+ @Override+ public Status needsField(FieldInfo fieldInfo) {+ return (matchRanges.containsKey(fieldInfo.name) || needsField.test(fieldInfo.name))+ ? Status.YES+ : Status.NO;+ }+ };leafReader.document(leafDocId, visitor);return visitor.getDocument();@@ -206,12 +206,16 @@ public class MatchHighlighter {}public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) {- this(searcher, analyzer, MatchRegionRetriever.computeOffsetRetrievalStrategies(searcher.getIndexReader(), analyzer));+ this(+ searcher,+ analyzer,+ MatchRegionRetriever.computeOffsetRetrievalStrategies(searcher.getIndexReader(), analyzer));}- public MatchHighlighter(IndexSearcher searcher,- Analyzer analyzer,- OffsetsRetrievalStrategySupplier offsetsRetrievalStrategies) {+ public MatchHighlighter(+ IndexSearcher searcher,+ Analyzer analyzer,+ OffsetsRetrievalStrategySupplier offsetsRetrievalStrategies) {this.searcher = searcher;this.offsetsRetrievalStrategies = offsetsRetrievalStrategies;this.analyzer = analyzer;@@ -229,8 +233,12 @@ public class MatchHighlighter {for (Query q : queries) {MatchRegionRetriever highlighter =new MatchRegionRetriever(searcher, searcher.rewrite(q), offsetsRetrievalStrategies);- highlighter.highlightDocuments(topDocs,- (int docId, LeafReader leafReader, int leafDocId, Map<String, List<OffsetRange>> hits) -> {+ highlighter.highlightDocuments(+ topDocs,+ (int docId,+ LeafReader leafReader,+ int leafDocId,+ Map<String, List<OffsetRange>> hits) -> {DocHit docHit = docHits.get(docId);if (docHit == null) {docHit = new DocHit(docId, leafReader, leafDocId);@@ -267,8 +275,9 @@ public class MatchHighlighter {List<OffsetRange> valueRanges = computeValueRanges(field, values);List<QueryOffsetRange> offsets = docHit.matchRanges.get(field);- List<String> formattedValues = fieldValueHighlighter(field, offsets != null)- .format(field, values, contiguousValue, valueRanges, offsets);+ List<String> formattedValues =+ fieldValueHighlighter(field, offsets != null)+ .format(field, values, contiguousValue, valueRanges, offsets);if (formattedValues != null) {docHighlights.fields.put(field, formattedValues);diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.javaindex 3f07aca6ff5..b71f37a27a6 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java@@ -16,6 +16,18 @@*/package org.apache.lucene.search.matchhighlight;+import java.io.IOException;+import java.util.Arrays;+import java.util.HashMap;+import java.util.HashSet;+import java.util.Iterator;+import java.util.List;+import java.util.Map;+import java.util.PrimitiveIterator;+import java.util.Set;+import java.util.TreeMap;+import java.util.TreeSet;+import java.util.function.Predicate;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.document.Document;import org.apache.lucene.index.FieldInfo;@@ -32,22 +44,9 @@ import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.Weight;-import java.io.IOException;-import java.util.Arrays;-import java.util.HashMap;-import java.util.HashSet;-import java.util.Iterator;-import java.util.List;-import java.util.Map;-import java.util.PrimitiveIterator;-import java.util.Set;-import java.util.TreeMap;-import java.util.TreeSet;-import java.util.function.Predicate;-/**- * Utility class to compute a list of "match regions" for a given query, searcher and- * document(s) using {@link Matches} API.+ * Utility class to compute a list of "match regions" for a given query, searcher and document(s)+ * using {@link Matches} API.*/public class MatchRegionRetriever {private final List<LeafReaderContext> leaves;@@ -58,20 +57,21 @@ public class MatchRegionRetriever {/*** A callback for accepting a single document (and its associated leaf reader, leaf document ID)- * and its match offset ranges, as indicated by the {@link Matches} interface retrieved for- * the query.+ * and its match offset ranges, as indicated by the {@link Matches} interface retrieved for the+ * query.*/@FunctionalInterfacepublic interface MatchOffsetsConsumer {- void accept(int docId, LeafReader leafReader, int leafDocId, Map<String, List<OffsetRange>> hits)+ void accept(+ int docId, LeafReader leafReader, int leafDocId, Map<String, List<OffsetRange>> hits)throws IOException;}/**- * An abstraction that provides document values for a given field. Default implementation- * in {@link DocumentFieldValueProvider} just reaches to a preloaded {@link Document}. It is- * possible to write a more efficient implementation on top of a reusable character buffer- * (that reuses the buffer while retrieving hit regions for documents).+ * An abstraction that provides document values for a given field. Default implementation in+ * {@link DocumentFieldValueProvider} just reaches to a preloaded {@link Document}. It is possible+ * to write a more efficient implementation on top of a reusable character buffer (that reuses the+ * buffer while retrieving hit regions for documents).*/@FunctionalInterfacepublic interface FieldValueProvider {@@ -81,23 +81,26 @@ public class MatchRegionRetriever {/*** A constructor with the default offset strategy supplier.*- * @param analyzer An analyzer that may be used to reprocess (retokenize) document fields- * in the absence of position offsets in the index. Note that the analyzer must return- * tokens (positions and offsets) identical to the ones stored in the index.+ * @param analyzer An analyzer that may be used to reprocess (retokenize) document fields in the+ * absence of position offsets in the index. Note that the analyzer must return tokens+ * (positions and offsets) identical to the ones stored in the index.*/- public MatchRegionRetriever(IndexSearcher searcher, Query query, Analyzer analyzer) throws IOException {+ public MatchRegionRetriever(IndexSearcher searcher, Query query, Analyzer analyzer)+ throws IOException {this(searcher, query, computeOffsetRetrievalStrategies(searcher.getIndexReader(), analyzer));}/*** @param searcher Index searcher to be used for retrieving matches.* @param query The query for which matches should be retrieved. The query should be rewritten- * against the provided searcher.- * @param fieldOffsetStrategySupplier A custom supplier of per-field {@link OffsetsRetrievalStrategy}- * instances.+ * against the provided searcher.+ * @param fieldOffsetStrategySupplier A custom supplier of per-field {@link+ * OffsetsRetrievalStrategy} instances.*/- public MatchRegionRetriever(IndexSearcher searcher, Query query,- OffsetsRetrievalStrategySupplier fieldOffsetStrategySupplier)+ public MatchRegionRetriever(+ IndexSearcher searcher,+ Query query,+ OffsetsRetrievalStrategySupplier fieldOffsetStrategySupplier)throws IOException {leaves = searcher.getIndexReader().leaves();assert checkOrderConsistency(leaves);@@ -138,19 +141,19 @@ public class MatchRegionRetriever {preloadFields.retainAll(affectedFields);}- public void highlightDocuments(TopDocs topDocs, MatchOffsetsConsumer consumer) throws IOException {- highlightDocuments(Arrays.stream(topDocs.scoreDocs)- .mapToInt(scoreDoc -> scoreDoc.doc)- .sorted()- .iterator(), consumer);+ public void highlightDocuments(TopDocs topDocs, MatchOffsetsConsumer consumer)+ throws IOException {+ highlightDocuments(+ Arrays.stream(topDocs.scoreDocs).mapToInt(scoreDoc -> scoreDoc.doc).sorted().iterator(),+ consumer);}/*** Low-level, high-efficiency method for highlighting large numbers of documents at once in a* streaming fashion.*- * @param docIds A stream of <em>sorted</em> document identifiers for which hit ranges should- * be returned.+ * @param docIds A stream of <em>sorted</em> document identifiers for which hit ranges should be+ * returned.* @param consumer A streaming consumer for document-hits pairs.*/public void highlightDocuments(PrimitiveIterator.OfInt docIds, MatchOffsetsConsumer consumer)@@ -187,7 +190,8 @@ public class MatchRegionRetriever {}highlights.clear();- highlightDocument(currentContext, contextRelativeDocId, documentSupplier, (field) -> true, highlights);+ highlightDocument(+ currentContext, contextRelativeDocId, documentSupplier, (field) -> true, highlights);consumer.accept(docId, currentContext.reader(), contextRelativeDocId, highlights);}}@@ -212,7 +216,8 @@ public class MatchRegionRetriever {if (acceptField.test(field)) {MatchesIterator matchesIterator = matches.getMatches(field);if (matchesIterator == null) {- // No matches on this field, even though the field was part of the query. This may be possible+ // No matches on this field, even though the field was part of the query. This may be+ // possible// with complex queries that source non-text fields (have no "hit regions" in any textual// representation). Skip.} else {@@ -241,8 +246,8 @@ public class MatchRegionRetriever {}/**- * Compute default strategies for retrieving offsets from {@link MatchesIterator}- * instances for a set of given fields.+ * Compute default strategies for retrieving offsets from {@link MatchesIterator} instances for a+ * set of given fields.*/public static OffsetsRetrievalStrategySupplier computeOffsetRetrievalStrategies(IndexReader reader, Analyzer analyzer) {@@ -274,22 +279,18 @@ public class MatchRegionRetriever {return new OffsetsFromTokens(field, analyzer);default:- return- (matchesIterator, doc) -> {- throw new IOException(- "Field is indexed without positions and/or offsets: "- + field- + ", "- + fieldInfo.getIndexOptions());- };+ return (matchesIterator, doc) -> {+ throw new IOException(+ "Field is indexed without positions and/or offsets: "+ + field+ + ", "+ + fieldInfo.getIndexOptions());+ };}};}- /**- * Implements {@link FieldValueProvider} wrapping a preloaded- * {@link Document}.- */+ /** Implements {@link FieldValueProvider} wrapping a preloaded {@link Document}. */private static final class DocumentFieldValueProvider implements FieldValueProvider {private final Document doc;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetRange.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetRange.javaindex 731650c2a38..6b5e79bb102 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetRange.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetRange.java@@ -18,9 +18,7 @@ package org.apache.lucene.search.matchhighlight;import java.util.Objects;-/**- * A non-empty range of offset positions.- */+/** A non-empty range of offset positions. */public class OffsetRange {/** Start index, inclusive. */public final int from;@@ -64,9 +62,8 @@ public class OffsetRange {}/**- * Returns a sub-range of this range (a copy). Subclasses should- * override and return an appropriate type covariant so that payloads- * are not lost.+ * Returns a sub-range of this range (a copy). Subclasses should override and return an+ * appropriate type covariant so that payloads are not lost.*/public OffsetRange slice(int from, int to) {assert from >= this.from;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.javaindex 1352776f9fa..ca270c0562b 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java@@ -16,15 +16,12 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.search.MatchesIterator;-import java.io.IOException;import java.util.ArrayList;import java.util.List;+import org.apache.lucene.search.MatchesIterator;-/**- * This strategy retrieves offsets directly from {@link MatchesIterator}.- */+/** This strategy retrieves offsets directly from {@link MatchesIterator}. */public final class OffsetsFromMatchIterator implements OffsetsRetrievalStrategy {private final String field;@@ -33,7 +30,8 @@ public final class OffsetsFromMatchIterator implements OffsetsRetrievalStrategy}@Override- public List<OffsetRange> get(MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)+ public List<OffsetRange> get(+ MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)throws IOException {ArrayList<OffsetRange> ranges = new ArrayList<>();while (matchesIterator.next()) {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.javaindex f9049f3a182..e35107ca708 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java@@ -16,16 +16,15 @@*/package org.apache.lucene.search.matchhighlight;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.search.MatchesIterator;-import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-/*** This strategy applies to fields with stored positions but no offsets. We re-analyze the field's* value to find out offsets of match positions.@@ -116,8 +115,8 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {if (position >= minPosition) {// Correct left and right offsets for each span this position applies to.- int startOffset = valueOffset + offsetAttr.startOffset();- int endOffset = valueOffset + offsetAttr.endOffset();+ int startOffset = valueOffset + offsetAttr.startOffset();+ int endOffset = valueOffset + offsetAttr.endOffset();int j = 0;for (int i = 0; i < spanCount; i++) {@@ -130,7 +129,7 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {// this span can't intersect with any following position// so omit it by skipping j++.continue;- }+ }}j++;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.javaindex 85c4f40ace7..913d89c7a1a 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java@@ -16,6 +16,11 @@*/package org.apache.lucene.search.matchhighlight;+import java.io.IOException;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Set;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;@@ -26,18 +31,12 @@ import org.apache.lucene.search.Query;import org.apache.lucene.search.QueryVisitor;import org.apache.lucene.util.BytesRef;-import java.io.IOException;-import java.util.ArrayList;-import java.util.HashSet;-import java.util.List;-import java.util.Set;-/**- * This strategy works for fields where we know the match occurred but there are- * no known positions or offsets.- * <p>- * We re-analyze field values and return offset ranges for returned tokens that- * are also returned by the query's term collector.+ * This strategy works for fields where we know the match occurred but there are no known positions+ * or offsets.+ *+ * <p>We re-analyze field values and return offset ranges for returned tokens that are also returned+ * by the query's term collector.*/public final class OffsetsFromTokens implements OffsetsRetrievalStrategy {private final String field;@@ -49,22 +48,25 @@ public final class OffsetsFromTokens implements OffsetsRetrievalStrategy {}@Override- public List<OffsetRange> get(MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc) throws IOException {+ public List<OffsetRange> get(+ MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)+ throws IOException {List<CharSequence> values = doc.getValues(field);Set<BytesRef> matchTerms = new HashSet<>();while (matchesIterator.next()) {Query q = matchesIterator.getQuery();- q.visit(new QueryVisitor() {- @Override- public void consumeTerms(Query query, Term... terms) {- for (Term t : terms) {- if (field.equals(t.field())) {- matchTerms.add(t.bytes());+ q.visit(+ new QueryVisitor() {+ @Override+ public void consumeTerms(Query query, Term... terms) {+ for (Term t : terms) {+ if (field.equals(t.field())) {+ matchTerms.add(t.bytes());+ }+ }}- }- }- });+ });}ArrayList<OffsetRange> ranges = new ArrayList<>();diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.javaindex d32650d4ffc..108fed577e9 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java@@ -16,22 +16,20 @@*/package org.apache.lucene.search.matchhighlight;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.search.MatchesIterator;-import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-/**- * This strategy works for fields where we know the match occurred but there are- * no known positions or offsets.- * <p>- * We re-analyze field values and return offset ranges for entire values- * (not individual tokens). Re-analysis is required because analyzer may return- * an unknown offset gap.+ * This strategy works for fields where we know the match occurred but there are no known positions+ * or offsets.+ *+ * <p>We re-analyze field values and return offset ranges for entire values (not individual tokens).+ * Re-analysis is required because analyzer may return an unknown offset gap.*/public final class OffsetsFromValues implements OffsetsRetrievalStrategy {private final String field;@@ -43,7 +41,9 @@ public final class OffsetsFromValues implements OffsetsRetrievalStrategy {}@Override- public List<OffsetRange> get(MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc) throws IOException {+ public List<OffsetRange> get(+ MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)+ throws IOException {List<CharSequence> values = doc.getValues(field);ArrayList<OffsetRange> ranges = new ArrayList<>();diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.javaindex 62eb25ff241..3cbd3d412c0 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java@@ -16,26 +16,22 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.search.MatchesIterator;-import java.io.IOException;import java.util.List;+import org.apache.lucene.search.MatchesIterator;/*** Determines how match offset regions are computed from {@link MatchesIterator}. Several- * possibilities exist, ranging from retrieving offsets directly from a match instance- * to re-evaluating the document's field and recomputing offsets from there.+ * possibilities exist, ranging from retrieving offsets directly from a match instance to+ * re-evaluating the document's field and recomputing offsets from there.*/public interface OffsetsRetrievalStrategy {- /**- * Return value offsets (match ranges) acquired from the given {@link MatchesIterator}.- */- List<OffsetRange> get(MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)+ /** Return value offsets (match ranges) acquired from the given {@link MatchesIterator}. */+ List<OffsetRange> get(+ MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)throws IOException;- /**- * Whether this strategy requires document field access.- */+ /** Whether this strategy requires document field access. */default boolean requiresDocument() {return false;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategySupplier.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategySupplier.javaindex 15f4c049037..c94036c80d0 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategySupplier.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategySupplier.java@@ -18,9 +18,7 @@ package org.apache.lucene.search.matchhighlight;import java.util.function.Function;-/**- * A per-field supplier of {@link OffsetsRetrievalStrategy}.- */+/** A per-field supplier of {@link OffsetsRetrievalStrategy}. */@FunctionalInterface-public interface OffsetsRetrievalStrategySupplier extends Function<String, OffsetsRetrievalStrategy> {-}+public interface OffsetsRetrievalStrategySupplier+ extends Function<String, OffsetsRetrievalStrategy> {}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/Passage.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/Passage.javaindex 7fafeddfb16..18dc9e1e8d3 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/Passage.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/Passage.java@@ -32,10 +32,7 @@ public class Passage extends OffsetRange {this.markers = markers;}- /**- * Passages can't be sliced as it could split previously determined- * highlight markers.- */+ /** Passages can't be sliced as it could split previously determined highlight markers. */@Overridepublic OffsetRange slice(int from, int to) {throw new RuntimeException("Passages.slice() does not make sense?");diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageAdjuster.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageAdjuster.javaindex 8415a2ed1cc..bc3f4debcbd 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageAdjuster.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageAdjuster.java@@ -17,11 +17,11 @@package org.apache.lucene.search.matchhighlight;/**- * Adjusts the range of one or more passages over a given value. An example- * adjuster could shift passage boundary to the next or previous word delimiter- * or white space, for example.+ * Adjusts the range of one or more passages over a given value. An example adjuster could shift+ * passage boundary to the next or previous word delimiter or white space, for example.*/public interface PassageAdjuster {void currentValue(CharSequence value);+OffsetRange adjust(Passage p);}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageFormatter.javaindex 2cb5b4e77b5..ff71237164f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageFormatter.java@@ -26,12 +26,11 @@ import java.util.RandomAccess;import java.util.function.Function;/**- * Formats a collection of {@linkplain Passage passages} over a given string, cleaning up- * and resolving restrictions concerning overlaps, allowed sub-ranges over the- * input string and length restrictions.+ * Formats a collection of {@linkplain Passage passages} over a given string, cleaning up and+ * resolving restrictions concerning overlaps, allowed sub-ranges over the input string and length+ * restrictions.*- * Passages are demarcated with constructor-provided ellipsis and start/end marker- * sequences.+ * <p>Passages are demarcated with constructor-provided ellipsis and start/end marker sequences.*/public class PassageFormatter {private final String ellipsis;@@ -94,13 +93,17 @@ public class PassageFormatter {}private boolean withinRange(OffsetRange limits, List<? extends OffsetRange> contained) {- contained.forEach(r -> {- if (r.from < limits.from || r.to > limits.to) {- throw new AssertionError(String.format(Locale.ROOT,- "Range outside of the permitted limit (limit = %s): %s",- limits, r));- }- });+ contained.forEach(+ r -> {+ if (r.from < limits.from || r.to > limits.to) {+ throw new AssertionError(+ String.format(+ Locale.ROOT,+ "Range outside of the permitted limit (limit = %s): %s",+ limits,+ r));+ }+ });return true;}@@ -131,9 +134,7 @@ public class PassageFormatter {return buf;}- /**- * Handle multiple markers, possibly overlapping or nested.- */+ /** Handle multiple markers, possibly overlapping or nested. */private void multipleMarkers(CharSequence value, final Passage p, StringBuilder b, ArrayList<OffsetRange> markerStack) {int at = p.from;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.javaindex 0468802b50e..286e0fe4e3a 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java@@ -16,9 +16,6 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.PriorityQueue;-import java.util.ArrayList;import java.util.Arrays;import java.util.Collections;@@ -26,6 +23,8 @@ import java.util.Comparator;import java.util.Iterator;import java.util.List;import java.util.RandomAccess;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.PriorityQueue;/** Selects fragments of text that score best for the given set of highlight markers. */public class PassageSelector {@@ -101,7 +100,7 @@ public class PassageSelector {markers = splitOrTruncateToWindows(markers, maxPassageWindow, permittedPassageRanges);// Sort markers by their start offset, shortest first.- markers.sort(Comparator.<OffsetRange> comparingInt(r -> r.from).thenComparingInt(r -> r.to));+ markers.sort(Comparator.<OffsetRange>comparingInt(r -> r.from).thenComparingInt(r -> r.to));final int max = markers.size();int markerIndex = 0;@@ -226,10 +225,11 @@ public class PassageSelector {return Arrays.asList(passages);}- /**- * Truncate or split highlight markers that cross permitted value boundaries.- */- private List<? extends OffsetRange> splitOrTruncateToWindows(List<? extends OffsetRange> markers, int maxPassageWindow, List<OffsetRange> permittedPassageRanges) {+ /** Truncate or split highlight markers that cross permitted value boundaries. */+ private List<? extends OffsetRange> splitOrTruncateToWindows(+ List<? extends OffsetRange> markers,+ int maxPassageWindow,+ List<OffsetRange> permittedPassageRanges) {// Process markers overlapping with each permitted window.ArrayList<OffsetRange> processedMarkers = new ArrayList<>(markers.size());for (OffsetRange marker : markers) {@@ -277,7 +277,10 @@ public class PassageSelector {* Invoked when no passages could be selected (due to constraints or lack of highlight markers).*/protected Passage[] pickDefaultPassage(- CharSequence value, int maxCharacterWindow, int maxPassages, List<OffsetRange> permittedPassageRanges) {+ CharSequence value,+ int maxCharacterWindow,+ int maxPassages,+ List<OffsetRange> permittedPassageRanges) {// Search for the first range that is not empty.ArrayList<Passage> defaultPassages = new ArrayList<>();for (OffsetRange o : permittedPassageRanges) {@@ -287,7 +290,11 @@ public class PassageSelector {int to = Math.min(value.length(), o.to);if (o.from < to) {- defaultPassages.add(new Passage(o.from, o.from + Math.min(maxCharacterWindow, o.length()), Collections.emptyList()));+ defaultPassages.add(+ new Passage(+ o.from,+ o.from + Math.min(maxCharacterWindow, o.length()),+ Collections.emptyList()));}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/package-info.javaindex 49af79fcc8b..d19e8930be0 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/package-info.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/package-info.java@@ -16,13 +16,13 @@*//**- * This package contains several components useful to build a highlighter- * on top of the {@link org.apache.lucene.search.Matches} API.+ * This package contains several components useful to build a highlighter on top of the {@link+ * org.apache.lucene.search.Matches} API.*- * {@link org.apache.lucene.search.matchhighlight.MatchRegionRetriever} can be- * used to retrieve hit areas for a given {@link org.apache.lucene.search.Query}- * and one (or more) indexed documents. These hit areas can be then passed to- * {@link org.apache.lucene.search.matchhighlight.PassageSelector} and formatted- * with {@link org.apache.lucene.search.matchhighlight.PassageFormatter}.+ * <p>{@link org.apache.lucene.search.matchhighlight.MatchRegionRetriever} can be used to retrieve+ * hit areas for a given {@link org.apache.lucene.search.Query} and one (or more) indexed documents.+ * These hit areas can be then passed to {@link+ * org.apache.lucene.search.matchhighlight.PassageSelector} and formatted with {@link+ * org.apache.lucene.search.matchhighlight.PassageFormatter}.*/package org.apache.lucene.search.matchhighlight;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/AnalysisOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/AnalysisOffsetStrategy.javaindex 63785e35f8a..180d5b0c4ec 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/AnalysisOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/AnalysisOffsetStrategy.java@@ -17,7 +17,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;@@ -25,9 +24,8 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;/**- * Provides a base class for analysis based offset strategies to extend from.- * Requires an Analyzer and provides an override-able method for altering how- * the TokenStream is created.+ * Provides a base class for analysis based offset strategies to extend from. Requires an Analyzer+ * and provides an override-able method for altering how the TokenStream is created.** @lucene.internal*/@@ -56,25 +54,34 @@ public abstract class AnalysisOffsetStrategy extends FieldOffsetStrategy {return analyzer.tokenStream(getField(), content);}- TokenStream subTokenStream = analyzer.tokenStream(getField(), content.substring(0, splitCharIdx));+ TokenStream subTokenStream =+ analyzer.tokenStream(getField(), content.substring(0, splitCharIdx));- return new MultiValueTokenStream(subTokenStream, getField(), analyzer, content, UnifiedHighlighter.MULTIVAL_SEP_CHAR, splitCharIdx);+ return new MultiValueTokenStream(+ subTokenStream,+ getField(),+ analyzer,+ content,+ UnifiedHighlighter.MULTIVAL_SEP_CHAR,+ splitCharIdx);}/**- * Wraps an {@link Analyzer} and string text that represents multiple values delimited by a specified character. This- * exposes a TokenStream that matches what would get indexed considering the- * {@link Analyzer#getPositionIncrementGap(String)}. Currently this assumes {@link Analyzer#getOffsetGap(String)} is- * 1; an exception will be thrown if it isn't.- * <br />- * It would be more orthogonal for this to be an Analyzer since we're wrapping an Analyzer but doing so seems like- * more work. The underlying components see a Reader not a String -- and the String is easy to- * split up without redundant buffering.+ * Wraps an {@link Analyzer} and string text that represents multiple values delimited by a+ * specified character. This exposes a TokenStream that matches what would get indexed considering+ * the {@link Analyzer#getPositionIncrementGap(String)}. Currently this assumes {@link+ * Analyzer#getOffsetGap(String)} is 1; an exception will be thrown if it isn't.+ *+ * <p>It would be more orthogonal for this to be an Analyzer since we're wrapping an Analyzer but+ * doing so seems like more work. The underlying components see a Reader not a String -- and the+ * String is easy to split up without redundant buffering.** @lucene.internal*/- // TODO we could make this go away. MemoryIndexOffsetStrategy could simply split and analyze each value into the- // MemoryIndex. TokenStreamOffsetStrategy's hack TokenStreamPostingsEnum could incorporate this logic,+ // TODO we could make this go away. MemoryIndexOffsetStrategy could simply split and analyze each+ // value into the+ // MemoryIndex. TokenStreamOffsetStrategy's hack TokenStreamPostingsEnum could incorporate this+ // logic,// albeit with less code, less hack.private static final class MultiValueTokenStream extends TokenFilter {@@ -83,15 +90,21 @@ public abstract class AnalysisOffsetStrategy extends FieldOffsetStrategy {private final String content;private final char splitChar;- private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute posIncAtt =+ addAttribute(PositionIncrementAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);private int startValIdx = 0;private int endValIdx;private int remainingPosInc = 0;- private MultiValueTokenStream(TokenStream subTokenStream, String fieldName, Analyzer indexAnalyzer,- String content, char splitChar, int splitCharIdx) {+ private MultiValueTokenStream(+ TokenStream subTokenStream,+ String fieldName,+ Analyzer indexAnalyzer,+ String content,+ char splitChar,+ int splitCharIdx) {super(subTokenStream); // subTokenStream is already initialized to operate on the first valuethis.fieldName = fieldName;this.indexAnalyzer = indexAnalyzer;@@ -115,19 +128,18 @@ public abstract class AnalysisOffsetStrategy extends FieldOffsetStrategy {if (input.incrementToken()) {// Position tracking:- if (remainingPosInc > 0) {//usually true first token of additional values (not first val)+ if (remainingPosInc > 0) {+ // usually true first token of additional values (not first val)posIncAtt.setPositionIncrement(remainingPosInc + posIncAtt.getPositionIncrement());- remainingPosInc = 0;//reset+ remainingPosInc = 0; // reset}// Offset tracking:offsetAtt.setOffset(- startValIdx + offsetAtt.startOffset(),- startValIdx + offsetAtt.endOffset()- );+ startValIdx + offsetAtt.startOffset(), startValIdx + offsetAtt.endOffset());return true;}- if (endValIdx == content.length()) {//no more+ if (endValIdx == content.length()) { // no morereturn false;}@@ -139,20 +151,27 @@ public abstract class AnalysisOffsetStrategy extends FieldOffsetStrategy {// Get new tokenStream based on next segment divided by the splitCharstartValIdx = endValIdx + 1;endValIdx = content.indexOf(splitChar, startValIdx);- if (endValIdx == -1) {//EOF+ if (endValIdx == -1) { // EOFendValIdx = content.length();}- TokenStream tokenStream = indexAnalyzer.tokenStream(fieldName, content.substring(startValIdx, endValIdx));- if (tokenStream != input) {// (input is defined in TokenFilter set in the constructor)- // This is a grand trick we do -- knowing that the analyzer's re-use strategy is going to produce the- // very same tokenStream instance and thus have the same AttributeSource as this wrapping TokenStream+ TokenStream tokenStream =+ indexAnalyzer.tokenStream(fieldName, content.substring(startValIdx, endValIdx));+ if (tokenStream != input) { // (input is defined in TokenFilter set in the constructor)+ // This is a grand trick we do -- knowing that the analyzer's re-use strategy is going to+ // produce the+ // very same tokenStream instance and thus have the same AttributeSource as this wrapping+ // TokenStream// since we used it as our input in the constructor.- // Were this not the case, we'd have to copy every attribute of interest since we can't alter the- // AttributeSource of this wrapping TokenStream post-construction (it's all private/final).- // If this is a problem, we could do that instead; maybe with a custom CharTermAttribute that allows+ // Were this not the case, we'd have to copy every attribute of interest since we can't+ // alter the+ // AttributeSource of this wrapping TokenStream post-construction (it's all+ // private/final).+ // If this is a problem, we could do that instead; maybe with a custom CharTermAttribute+ // that allows// us to easily set the char[] reference without literally copying char by char.- throw new IllegalStateException("Require TokenStream re-use. Unsupported re-use strategy?: " +- indexAnalyzer.getReuseStrategy());+ throw new IllegalStateException(+ "Require TokenStream re-use. Unsupported re-use strategy?: "+ + indexAnalyzer.getReuseStrategy());}tokenStream.reset();} // while loop to increment token of this new value@@ -163,9 +182,7 @@ public abstract class AnalysisOffsetStrategy extends FieldOffsetStrategy {super.end();// Offset tracking:offsetAtt.setOffset(- startValIdx + offsetAtt.startOffset(),- startValIdx + offsetAtt.endOffset());+ startValIdx + offsetAtt.startOffset(), startValIdx + offsetAtt.endOffset());}-}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CharArrayMatcher.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CharArrayMatcher.javaindex 75d5606e049..596e3a3e2d6 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CharArrayMatcher.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CharArrayMatcher.java@@ -18,7 +18,6 @@package org.apache.lucene.search.uhighlight;import java.util.List;-import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRef;import org.apache.lucene.util.automaton.Automata;@@ -31,14 +30,10 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton;*/public interface CharArrayMatcher {- /**- * Return {@code true} if the passed-in character array matches- */+ /** Return {@code true} if the passed-in character array matches */boolean match(char[] s, int offset, int length);- /**- * Return {@code true} if the passed-in CharsRef matches- */+ /** Return {@code true} if the passed-in CharsRef matches */default boolean match(CharsRef chars) {return match(chars.chars, chars.offset, chars.length);}@@ -47,5 +42,4 @@ public interface CharArrayMatcher {CharacterRunAutomaton a = new CharacterRunAutomaton(Automata.makeStringUnion(terms));return a::run;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.javaindex 7119aed9684..7274c501447 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/CustomSeparatorBreakIterator.java@@ -20,7 +20,8 @@ import java.text.BreakIterator;import java.text.CharacterIterator;/**- * A {@link BreakIterator} that breaks the text whenever a certain separator, provided as a constructor argument, is found.+ * A {@link BreakIterator} that breaks the text whenever a certain separator, provided as a+ * constructor argument, is found.*/public final class CustomSeparatorBreakIterator extends BreakIterator {@@ -74,7 +75,8 @@ public final class CustomSeparatorBreakIterator extends BreakIterator {if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {throw new IllegalArgumentException("offset out of bounds");} else if (pos == text.getEndIndex()) {- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in+ // something)// https://bugs.openjdk.java.net/browse/JDK-8015110text.setIndex(text.getEndIndex());current = text.getIndex();@@ -111,7 +113,8 @@ public final class CustomSeparatorBreakIterator extends BreakIterator {if (pos < text.getBeginIndex() || pos > text.getEndIndex()) {throw new IllegalArgumentException("offset out of bounds");} else if (pos == text.getBeginIndex()) {- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in+ // something)// https://bugs.openjdk.java.net/browse/JDK-8015110text.setIndex(text.getBeginIndex());current = text.getIndex();diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/DefaultPassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/DefaultPassageFormatter.javaindex 701197aa980..49b88f8bffd 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/DefaultPassageFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/DefaultPassageFormatter.java@@ -16,12 +16,11 @@*/package org.apache.lucene.search.uhighlight;-/*** Creates a formatted snippet from the top passages.- * <p>- * The default implementation marks the query terms as bold, and places- * ellipses between unconnected passages.+ *+ * <p>The default implementation marks the query terms as bold, and places ellipses between+ * unconnected passages.*/public class DefaultPassageFormatter extends PassageFormatter {/** text that will appear before highlighted terms */@@ -33,9 +32,7 @@ public class DefaultPassageFormatter extends PassageFormatter {/** true if we should escape for html */protected final boolean escape;- /**- * Creates a new DefaultPassageFormatter with the default tags.- */+ /** Creates a new DefaultPassageFormatter with the default tags. */public DefaultPassageFormatter() {this("<b>", "</b>", "... ", false);}@@ -43,10 +40,10 @@ public class DefaultPassageFormatter extends PassageFormatter {/*** Creates a new DefaultPassageFormatter with custom tags.*- * @param preTag text which should appear before a highlighted term.- * @param postTag text which should appear after a highlighted term.+ * @param preTag text which should appear before a highlighted term.+ * @param postTag text which should appear after a highlighted term.* @param ellipsis text which should be used to connect two unconnected passages.- * @param escape true if text should be html-escaped+ * @param escape true if text should be html-escaped*/public DefaultPassageFormatter(String preTag, String postTag, String ellipsis, boolean escape) {if (preTag == null || postTag == null || ellipsis == null) {@@ -71,14 +68,14 @@ public class DefaultPassageFormatter extends PassageFormatter {for (int i = 0; i < passage.getNumMatches(); i++) {int start = passage.getMatchStarts()[i];assert start >= pos && start < passage.getEndOffset();- //append content before this start+ // append content before this startappend(sb, content, pos, start);int end = passage.getMatchEnds()[i];assert end > start;// its possible to have overlapping terms.// Look ahead to expand 'end' past all overlapping:- while (i + 1 < passage.getNumMatches() && passage.getMatchStarts()[i+1] < end) {+ while (i + 1 < passage.getNumMatches() && passage.getMatchStarts()[i + 1] < end) {end = passage.getMatchEnds()[++i];}end = Math.min(end, passage.getEndOffset()); // in case match straddles past passage@@ -99,10 +96,10 @@ public class DefaultPassageFormatter extends PassageFormatter {/*** Appends original text to the response.*- * @param dest resulting text, possibly transformed or encoded+ * @param dest resulting text, possibly transformed or encoded* @param content original text content- * @param start index of the first character in content- * @param end index of the character following the last character in content+ * @param start index of the first character in content+ * @param end index of the character following the last character in content*/protected void append(StringBuilder dest, String content, int start, int end) {if (escape) {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.javaindex 5e45e3f40a9..aacb9089e91 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldHighlighter.java@@ -23,7 +23,6 @@ import java.util.Arrays;import java.util.Comparator;import java.util.List;import java.util.PriorityQueue;-import org.apache.lucene.index.LeafReader;import org.apache.lucene.util.BytesRef;@@ -42,9 +41,14 @@ public class FieldHighlighter {protected final int maxNoHighlightPassages;protected final PassageFormatter passageFormatter;- public FieldHighlighter(String field, FieldOffsetStrategy fieldOffsetStrategy, BreakIterator breakIterator,- PassageScorer passageScorer, int maxPassages, int maxNoHighlightPassages,- PassageFormatter passageFormatter) {+ public FieldHighlighter(+ String field,+ FieldOffsetStrategy fieldOffsetStrategy,+ BreakIterator breakIterator,+ PassageScorer passageScorer,+ int maxPassages,+ int maxNoHighlightPassages,+ PassageFormatter passageFormatter) {this.field = field;this.fieldOffsetStrategy = fieldOffsetStrategy;this.breakIterator = breakIterator;@@ -62,11 +66,11 @@ public class FieldHighlighter {return fieldOffsetStrategy.getOffsetSource();}- /**- * The primary method -- highlight this doc, assuming a specific field and given this content.- */- public Object highlightFieldForDoc(LeafReader reader, int docId, String content) throws IOException {- // note: it'd be nice to accept a CharSequence for content, but we need a CharacterIterator impl for it.+ /** The primary method -- highlight this doc, assuming a specific field and given this content. */+ public Object highlightFieldForDoc(LeafReader reader, int docId, String content)+ throws IOException {+ // note: it'd be nice to accept a CharSequence for content, but we need a CharacterIterator impl+ // for it.if (content.length() == 0) {return null; // nothing to do}@@ -76,12 +80,14 @@ public class FieldHighlighter {try (OffsetsEnum offsetsEnums = fieldOffsetStrategy.getOffsetsEnum(reader, docId, content)) {// Highlight the offsetsEnum list against the content to produce Passages.- Passage[] passages = highlightOffsetsEnums(offsetsEnums);// and breakIterator & scorer+ Passage[] passages = highlightOffsetsEnums(offsetsEnums); // and breakIterator & scorer// Format the resulting Passages.if (passages.length == 0) {// no passages were returned, so ask for a default summary- passages = getSummaryPassagesNoHighlight(maxNoHighlightPassages == -1 ? maxPassages : maxNoHighlightPassages);+ passages =+ getSummaryPassagesNoHighlight(+ maxNoHighlightPassages == -1 ? maxPassages : maxNoHighlightPassages);}if (passages.length > 0) {@@ -93,10 +99,9 @@ public class FieldHighlighter {}/**- * Called to summarize a document when no highlights were found.- * By default this just returns the first- * {@link #maxPassages} sentences; subclasses can override to customize.- * The state of {@link #breakIterator} should be at the beginning.+ * Called to summarize a document when no highlights were found. By default this just returns the+ * first {@link #maxPassages} sentences; subclasses can override to customize. The state of {@link+ * #breakIterator} should be at the beginning.*/protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) {assert breakIterator.current() == breakIterator.first();@@ -122,8 +127,7 @@ public class FieldHighlighter {// algorithm: treat sentence snippets as miniature documents// we can intersect these with the postings lists via BreakIterator.preceding(offset),s// score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))- protected Passage[] highlightOffsetsEnums(OffsetsEnum off)- throws IOException {+ protected Passage[] highlightOffsetsEnums(OffsetsEnum off) throws IOException {final int contentLength = this.breakIterator.getText().getEndIndex();@@ -131,22 +135,27 @@ public class FieldHighlighter {return new Passage[0];}- PriorityQueue<Passage> passageQueue = new PriorityQueue<>(Math.min(64, maxPassages + 1), (left, right) -> {- if (left.getScore() < right.getScore()) {- return -1;- } else if (left.getScore() > right.getScore()) {- return 1;- } else {- return left.getStartOffset() - right.getStartOffset();- }- });- Passage passage = new Passage(); // the current passage in-progress. Will either get reset or added to queue.+ PriorityQueue<Passage> passageQueue =+ new PriorityQueue<>(+ Math.min(64, maxPassages + 1),+ (left, right) -> {+ if (left.getScore() < right.getScore()) {+ return -1;+ } else if (left.getScore() > right.getScore()) {+ return 1;+ } else {+ return left.getStartOffset() - right.getStartOffset();+ }+ });+ Passage passage =+ new Passage(); // the current passage in-progress. Will either get reset or added to queue.int lastPassageEnd = 0;do {int start = off.startOffset();if (start == -1) {- throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");+ throw new IllegalArgumentException(+ "field '" + field + "' was indexed without offsets, cannot highlight");}int end = off.endOffset();if (start < contentLength && end > contentLength) {@@ -159,15 +168,23 @@ public class FieldHighlighter {if (start >= contentLength) {break;}- // find fragment from the middle of the match, so the result's length may be closer to fragsize+ // find fragment from the middle of the match, so the result's length may be closer to+ // fragsizefinal int center = start + (end - start) / 2;// advance breakIterator- passage.setStartOffset(Math.min(start, Math.max(this.breakIterator.preceding(Math.max(start + 1, center)), lastPassageEnd)));- lastPassageEnd = Math.max(end, Math.min(this.breakIterator.following(Math.min(end - 1, center)), contentLength));+ passage.setStartOffset(+ Math.min(+ start,+ Math.max(+ this.breakIterator.preceding(Math.max(start + 1, center)), lastPassageEnd)));+ lastPassageEnd =+ Math.max(+ end,+ Math.min(this.breakIterator.following(Math.min(end - 1, center)), contentLength));passage.setEndOffset(lastPassageEnd);}// Add this term to the passage.- BytesRef term = off.getTerm();// a reference; safe to refer to+ BytesRef term = off.getTerm(); // a reference; safe to refer toassert term != null;passage.addMatch(start, end, term, off.freq());} while (off.nextPosition());@@ -179,7 +196,11 @@ public class FieldHighlighter {return passages;}- private Passage maybeAddPassage(PriorityQueue<Passage> passageQueue, PassageScorer scorer, Passage passage, int contentLength) {+ private Passage maybeAddPassage(+ PriorityQueue<Passage> passageQueue,+ PassageScorer scorer,+ Passage passage,+ int contentLength) {if (passage.getStartOffset() == -1) {// empty passage, we can ignore itreturn passage;@@ -199,5 +220,4 @@ public class FieldHighlighter {}return passage;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.javaindex d7c936f41cc..dcc8e5af873 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java@@ -19,7 +19,6 @@ package org.apache.lucene.search.uhighlight;import java.io.IOException;import java.util.ArrayList;import java.util.List;-import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.PostingsEnum;@@ -33,8 +32,8 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRefBuilder;/**- * Ultimately returns an {@link OffsetsEnum} yielding potentially highlightable words in the text. Needs- * information about the query up front.+ * Ultimately returns an {@link OffsetsEnum} yielding potentially highlightable words in the text.+ * Needs information about the query up front.** @lucene.internal*/@@ -55,11 +54,13 @@ public abstract class FieldOffsetStrategy {/*** The primary method -- return offsets for highlightable words in the specified document.*- * Callers are expected to close the returned OffsetsEnum when it has been finished with+ * <p>Callers are expected to close the returned OffsetsEnum when it has been finished with*/- public abstract OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException;+ public abstract OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException;- protected OffsetsEnum createOffsetsEnumFromReader(LeafReader leafReader, int doc) throws IOException {+ protected OffsetsEnum createOffsetsEnumFromReader(LeafReader leafReader, int doc)+ throws IOException {final Terms termsIndex = leafReader.terms(getField());if (termsIndex == null) {return OffsetsEnum.EMPTY;@@ -80,7 +81,8 @@ public abstract class FieldOffsetStrategy {final BytesRef[] terms = components.getTerms();if (phraseHelper.hasPositionSensitivity()) {insensitiveTerms = phraseHelper.getAllPositionInsensitiveTerms();- assert insensitiveTerms.length <= terms.length : "insensitive terms should be smaller set of all terms";+ assert insensitiveTerms.length <= terms.length+ : "insensitive terms should be smaller set of all terms";} else {insensitiveTerms = terms;}@@ -100,41 +102,49 @@ public abstract class FieldOffsetStrategy {}switch (offsetsEnums.size()) {- case 0: return OffsetsEnum.EMPTY;- case 1: return offsetsEnums.get(0);- default: return new OffsetsEnum.MultiOffsetsEnum(offsetsEnums);+ case 0:+ return OffsetsEnum.EMPTY;+ case 1:+ return offsetsEnums.get(0);+ default:+ return new OffsetsEnum.MultiOffsetsEnum(offsetsEnums);}}- protected void createOffsetsEnumsWeightMatcher(LeafReader _leafReader, int docId, List<OffsetsEnum> results) throws IOException {+ protected void createOffsetsEnumsWeightMatcher(+ LeafReader _leafReader, int docId, List<OffsetsEnum> results) throws IOException {// remap fieldMatcher/requireFieldMatch fields to the field we are highlighting- LeafReader leafReader = new FilterLeafReader(_leafReader) {- @Override- public Terms terms(String field) throws IOException {- if (components.getFieldMatcher().test(field)) {- return super.terms(components.getField());- } else {- return super.terms(field);- }- }+ LeafReader leafReader =+ new FilterLeafReader(_leafReader) {+ @Override+ public Terms terms(String field) throws IOException {+ if (components.getFieldMatcher().test(field)) {+ return super.terms(components.getField());+ } else {+ return super.terms(field);+ }+ }- // So many subclasses do this!- //these ought to be a default or added via some intermediary like "FilterTransientLeafReader" (exception on close).- @Override- public CacheHelper getCoreCacheHelper() {- return null;- }+ // So many subclasses do this!+ // these ought to be a default or added via some intermediary like+ // "FilterTransientLeafReader" (exception on close).+ @Override+ public CacheHelper getCoreCacheHelper() {+ return null;+ }- @Override- public CacheHelper getReaderCacheHelper() {- return null;- }- };+ @Override+ public CacheHelper getReaderCacheHelper() {+ return null;+ }+ };IndexSearcher indexSearcher = new IndexSearcher(leafReader);indexSearcher.setQueryCache(null);- Matches matches = indexSearcher.rewrite(components.getQuery())- .createWeight(indexSearcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f)- .matches(leafReader.getContext(), docId);+ Matches matches =+ indexSearcher+ .rewrite(components.getQuery())+ .createWeight(indexSearcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f)+ .matches(leafReader.getContext(), docId);if (matches == null) {return; // doc doesn't match}@@ -147,17 +157,19 @@ public abstract class FieldOffsetStrategy {results.add(new OffsetsEnum.OfMatchesIteratorWithSubs(iterator));}}-}- protected void createOffsetsEnumsForTerms(BytesRef[] sourceTerms, Terms termsIndex, int doc, List<OffsetsEnum> results) throws IOException {- TermsEnum termsEnum = termsIndex.iterator();//does not return null+ protected void createOffsetsEnumsForTerms(+ BytesRef[] sourceTerms, Terms termsIndex, int doc, List<OffsetsEnum> results)+ throws IOException {+ TermsEnum termsEnum = termsIndex.iterator(); // does not return nullfor (BytesRef term : sourceTerms) {if (termsEnum.seekExact(term)) {PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS);if (postingsEnum == null) {// no offsets or positions available- throw new IllegalArgumentException("field '" + getField() + "' was indexed without offsets, cannot highlight");+ throw new IllegalArgumentException(+ "field '" + getField() + "' was indexed without offsets, cannot highlight");}if (doc == postingsEnum.advance(doc)) { // now it's positioned, although may be exhaustedresults.add(new OffsetsEnum.OfPostings(term, postingsEnum));@@ -166,7 +178,8 @@ public abstract class FieldOffsetStrategy {}}- protected void createOffsetsEnumsForAutomata(Terms termsIndex, int doc, List<OffsetsEnum> results) throws IOException {+ protected void createOffsetsEnumsForAutomata(Terms termsIndex, int doc, List<OffsetsEnum> results)+ throws IOException {final LabelledCharArrayMatcher[] automata = components.getAutomata();List<List<PostingsEnum>> automataPostings = new ArrayList<>(automata.length);for (int i = 0; i < automata.length; i++) {@@ -206,7 +219,5 @@ public abstract class FieldOffsetStrategy {results.add(new OffsetsEnum.OfPostings(wildcardTerm, sumFreq, postingsEnum));}}-}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.javaindex c2a50aee9d0..ff96916f77c 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LabelledCharArrayMatcher.java@@ -21,20 +21,17 @@ import org.apache.lucene.util.UnicodeUtil;import org.apache.lucene.util.automaton.ByteRunAutomaton;/**- * Associates a label with a CharArrayMatcher to distinguish different sources for terms in highlighting+ * Associates a label with a CharArrayMatcher to distinguish different sources for terms in+ * highlighting** @lucene.internal*/public interface LabelledCharArrayMatcher extends CharArrayMatcher {- /**- * @return the label for this matcher- */+ /** @return the label for this matcher */String getLabel();- /**- * Associates a label with a CharArrayMatcher- */+ /** Associates a label with a CharArrayMatcher */static LabelledCharArrayMatcher wrap(String label, CharArrayMatcher in) {return new LabelledCharArrayMatcher() {@Override@@ -49,40 +46,39 @@ public interface LabelledCharArrayMatcher extends CharArrayMatcher {};}- /**- * Returns a representation of the automaton that matches char[] instead of byte[]- */+ /** Returns a representation of the automaton that matches char[] instead of byte[] */static LabelledCharArrayMatcher wrap(String label, ByteRunAutomaton runAutomaton) {- return wrap(label, (chars, offset, length) -> {- int state = 0;- final int maxIdx = offset + length;- for (int i = offset; i < maxIdx; i++) {- final int code = chars[i];- int b;- // UTF16 to UTF8 (inlined logic from UnicodeUtil.UTF16toUTF8 )- if (code < 0x80) {- state = runAutomaton.step(state, code);- if (state == -1) return false;- } else if (code < 0x800) {- b = (0xC0 | (code >> 6));- state = runAutomaton.step(state, b);- if (state == -1) return false;- b = (0x80 | (code & 0x3F));- state = runAutomaton.step(state, b);- if (state == -1) return false;- } else {- // more complex- byte[] utf8Bytes = new byte[4 * (maxIdx - i)];- int utf8Len = UnicodeUtil.UTF16toUTF8(chars, i, maxIdx - i, utf8Bytes);- for (int utfIdx = 0; utfIdx < utf8Len; utfIdx++) {- state = runAutomaton.step(state, utf8Bytes[utfIdx] & 0xFF);+ return wrap(+ label,+ (chars, offset, length) -> {+ int state = 0;+ final int maxIdx = offset + length;+ for (int i = offset; i < maxIdx; i++) {+ final int code = chars[i];+ int b;+ // UTF16 to UTF8 (inlined logic from UnicodeUtil.UTF16toUTF8 )+ if (code < 0x80) {+ state = runAutomaton.step(state, code);if (state == -1) return false;+ } else if (code < 0x800) {+ b = (0xC0 | (code >> 6));+ state = runAutomaton.step(state, b);+ if (state == -1) return false;+ b = (0x80 | (code & 0x3F));+ state = runAutomaton.step(state, b);+ if (state == -1) return false;+ } else {+ // more complex+ byte[] utf8Bytes = new byte[4 * (maxIdx - i)];+ int utf8Len = UnicodeUtil.UTF16toUTF8(chars, i, maxIdx - i, utf8Bytes);+ for (int utfIdx = 0; utfIdx < utf8Len; utfIdx++) {+ state = runAutomaton.step(state, utf8Bytes[utfIdx] & 0xFF);+ if (state == -1) return false;+ }+ break;}- break;}- }- return runAutomaton.isAccept(state);- });+ return runAutomaton.isAccept(state);+ });}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LengthGoalBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LengthGoalBreakIterator.javaindex a4b5a07f63b..1f9fe1e8a19 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LengthGoalBreakIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/LengthGoalBreakIterator.java@@ -21,11 +21,12 @@ import java.text.BreakIterator;import java.text.CharacterIterator;/**- * Wraps another {@link BreakIterator} to skip past breaks that would result in passages that are too- * short. It's still possible to get a short passage but only at the very end of the input text.- * <p>- * Important: This is not a general purpose {@link BreakIterator}; it's only designed to work in a way- * compatible with the {@link UnifiedHighlighter}. Some assumptions are checked with Java assertions.+ * Wraps another {@link BreakIterator} to skip past breaks that would result in passages that are+ * too short. It's still possible to get a short passage but only at the very end of the input text.+ *+ * <p>Important: This is not a general purpose {@link BreakIterator}; it's only designed to work in+ * a way compatible with the {@link UnifiedHighlighter}. Some assumptions are checked with Java+ * assertions.** @lucene.experimental*/@@ -33,42 +34,56 @@ public class LengthGoalBreakIterator extends BreakIterator {private final BreakIterator baseIter;private final int lengthGoal;- private final float fragmentAlignment; // how much text to align before match-fragment, valid in range [0, 1]- private final boolean isMinimumLength; // if false then is "closest to" length+ // how much text to align before match-fragment, valid in range [0, 1]+ private final float fragmentAlignment;+ // if false then is "closest to" length+ private final boolean isMinimumLength;private int currentCache;/**- * Breaks will be at least {@code minLength} apart (to the extent possible),- * while trying to position the match inside the fragment according to {@code fragmentAlignment}.+ * Breaks will be at least {@code minLength} apart (to the extent possible), while trying to+ * position the match inside the fragment according to {@code fragmentAlignment}.*/- public static LengthGoalBreakIterator createMinLength(BreakIterator baseIter, int minLength,- float fragmentAlignment) {- return new LengthGoalBreakIterator(baseIter, minLength, fragmentAlignment, true, baseIter.current());+ public static LengthGoalBreakIterator createMinLength(+ BreakIterator baseIter, int minLength, float fragmentAlignment) {+ return new LengthGoalBreakIterator(+ baseIter, minLength, fragmentAlignment, true, baseIter.current());}- /** For backwards compatibility you can initialise the break iterator without fragmentAlignment. */+ /**+ * For backwards compatibility you can initialise the break iterator without fragmentAlignment.+ */@Deprecatedpublic static LengthGoalBreakIterator createMinLength(BreakIterator baseIter, int minLength) {return createMinLength(baseIter, minLength, 0.f);}/**- * Breaks will be on average {@code targetLength} apart; the closest break to this target (before or after)- * is chosen. The match will be positioned according to {@code fragmentAlignment} as much as possible.+ * Breaks will be on average {@code targetLength} apart; the closest break to this target (before+ * or after) is chosen. The match will be positioned according to {@code fragmentAlignment} as+ * much as possible.*/- public static LengthGoalBreakIterator createClosestToLength(BreakIterator baseIter, int targetLength,- float fragmentAlignment) {- return new LengthGoalBreakIterator(baseIter, targetLength, fragmentAlignment, false, baseIter.current());+ public static LengthGoalBreakIterator createClosestToLength(+ BreakIterator baseIter, int targetLength, float fragmentAlignment) {+ return new LengthGoalBreakIterator(+ baseIter, targetLength, fragmentAlignment, false, baseIter.current());}- /** For backwards compatibility you can initialise the break iterator without fragmentAlignment. */+ /**+ * For backwards compatibility you can initialise the break iterator without fragmentAlignment.+ */@Deprecated- public static LengthGoalBreakIterator createClosestToLength(BreakIterator baseIter, int targetLength) {+ public static LengthGoalBreakIterator createClosestToLength(+ BreakIterator baseIter, int targetLength) {return createClosestToLength(baseIter, targetLength, 0.f);}- private LengthGoalBreakIterator(BreakIterator baseIter, int lengthGoal, float fragmentAlignment,- boolean isMinimumLength, int currentCache) {+ private LengthGoalBreakIterator(+ BreakIterator baseIter,+ int lengthGoal,+ float fragmentAlignment,+ boolean isMinimumLength,+ int currentCache) {this.baseIter = baseIter;this.currentCache = currentCache;this.lengthGoal = lengthGoal;@@ -87,15 +102,26 @@ public class LengthGoalBreakIterator extends BreakIterator {@Overridepublic String toString() {String goalDesc = isMinimumLength ? "minLen" : "targetLen";- return getClass().getSimpleName() + "{" + goalDesc + "=" + lengthGoal + ", fragAlign=" + fragmentAlignment +- ", baseIter=" + baseIter + "}";+ return getClass().getSimpleName()+ + "{"+ + goalDesc+ + "="+ + lengthGoal+ + ", fragAlign="+ + fragmentAlignment+ + ", baseIter="+ + baseIter+ + "}";}@Overridepublic Object clone() {return new LengthGoalBreakIterator(- (BreakIterator) baseIter.clone(), lengthGoal, fragmentAlignment, isMinimumLength, currentCache- );+ (BreakIterator) baseIter.clone(),+ lengthGoal,+ fragmentAlignment,+ isMinimumLength,+ currentCache);}@Override@@ -150,7 +176,8 @@ public class LengthGoalBreakIterator extends BreakIterator {@Overridepublic int following(int matchEndIndex) {- return following(matchEndIndex, (matchEndIndex + 1) + (int)(lengthGoal * (1.f - fragmentAlignment)));+ return following(+ matchEndIndex, (matchEndIndex + 1) + (int) (lengthGoal * (1.f - fragmentAlignment)));}private int following(int matchEndIndex, int targetIdx) {@@ -185,7 +212,7 @@ public class LengthGoalBreakIterator extends BreakIterator {// called at start of new Passage given first word start offset@Overridepublic int preceding(int matchStartIndex) {- final int targetIdx = (matchStartIndex - 1) - (int)(lengthGoal * fragmentAlignment);+ final int targetIdx = (matchStartIndex - 1) - (int) (lengthGoal * fragmentAlignment);if (targetIdx <= 0) {if (currentCache == baseIter.first()) {return DONE;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.javaindex e67cef3a789..fd95bd9d7ac 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java@@ -21,7 +21,6 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.Collections;import java.util.List;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.FilteringTokenFilter;import org.apache.lucene.analysis.TokenStream;@@ -30,7 +29,6 @@ import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.memory.MemoryIndex;import org.apache.lucene.search.spans.SpanQuery;-/*** Uses an {@link Analyzer} on content to get offsets and then populates a {@link MemoryIndex}.*@@ -44,16 +42,16 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {public MemoryIndexOffsetStrategy(UHComponents components, Analyzer analyzer) {super(components, analyzer);- boolean storePayloads = components.getPhraseHelper().hasPositionSensitivity(); // might be needed- memoryIndex = new MemoryIndex(true, storePayloads);//true==store offsets- memIndexLeafReader = (LeafReader) memoryIndex.createSearcher().getIndexReader(); // appears to be re-usable+ boolean storePayloads =+ components.getPhraseHelper().hasPositionSensitivity(); // might be needed+ memoryIndex = new MemoryIndex(true, storePayloads); // true==store offsets+ memIndexLeafReader =+ (LeafReader) memoryIndex.createSearcher().getIndexReader(); // appears to be re-usable// preFilter for MemoryIndexpreMemIndexFilterAutomaton = buildCombinedAutomaton(components);}- /**- * Build one {@link CharArrayMatcher} matching any term the query might match.- */+ /** Build one {@link CharArrayMatcher} matching any term the query might match. */private static CharArrayMatcher buildCombinedAutomaton(UHComponents components) {// We don't know enough about the query to do this confidentlyif (components.getTerms() == null || components.getAutomata() == null) {@@ -66,20 +64,25 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {}Collections.addAll(allAutomata, components.getAutomata());for (SpanQuery spanQuery : components.getPhraseHelper().getSpanQueries()) {- Collections.addAll(allAutomata,- MultiTermHighlighting.extractAutomata(spanQuery, components.getFieldMatcher(), true));//true==lookInSpan+ Collections.addAll(+ allAutomata,+ MultiTermHighlighting.extractAutomata(+ spanQuery, components.getFieldMatcher(), true)); // true==lookInSpan}if (allAutomata.size() == 1) {return allAutomata.get(0);}- //TODO it'd be nice if we could get at the underlying Automaton in CharacterRunAutomaton so that we- // could union them all. But it's not exposed, and sometimes the automaton is byte (not char) oriented+ // TODO it'd be nice if we could get at the underlying Automaton in CharacterRunAutomaton so+ // that we+ // could union them all. But it's not exposed, and sometimes the automaton is byte (not char)+ // oriented// Return an aggregate CharArrayMatcher of othersreturn (chars, offset, length) -> {- for (int i = 0; i < allAutomata.size(); i++) {// don't use foreach to avoid Iterator allocation+ for (int i = 0; i < allAutomata.size(); i++) {+ // don't use foreach to avoid Iterator allocationif (allAutomata.get(i).match(chars, offset, length)) {return true;}@@ -89,7 +92,8 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {// note: don't need LimitTokenOffsetFilter since content is already truncated to maxLengthTokenStream tokenStream = tokenStream(content);@@ -98,24 +102,21 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {tokenStream = newKeepWordFilter(tokenStream, preMemIndexFilterAutomaton);}memoryIndex.reset();- memoryIndex.addField(getField(), tokenStream);//note: calls tokenStream.reset() & close()+ memoryIndex.addField(getField(), tokenStream); // note: calls tokenStream.reset() & close()if (reader == null) {return createOffsetsEnumFromReader(memIndexLeafReader, 0);} else {return createOffsetsEnumFromReader(- new OverlaySingleDocTermsLeafReader(- reader,- memIndexLeafReader,- getField(),- docId),+ new OverlaySingleDocTermsLeafReader(reader, memIndexLeafReader, getField(), docId),docId);}}- private static FilteringTokenFilter newKeepWordFilter(final TokenStream tokenStream,- final CharArrayMatcher matcher) {- // it'd be nice to use KeepWordFilter but it demands a CharArraySet. TODO File JIRA? Need a new interface?+ private static FilteringTokenFilter newKeepWordFilter(+ final TokenStream tokenStream, final CharArrayMatcher matcher) {+ // it'd be nice to use KeepWordFilter but it demands a CharArraySet. TODO File JIRA? Need a new+ // interface?return new FilteringTokenFilter(tokenStream) {final CharTermAttribute charAtt = addAttribute(CharTermAttribute.class);@@ -125,5 +126,4 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {}};}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.javaindex 02bd672c7b9..ca4ab04a040 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiTermHighlighting.java@@ -20,7 +20,6 @@ import java.util.ArrayList;import java.util.List;import java.util.function.Predicate;import java.util.function.Supplier;-import org.apache.lucene.search.AutomatonQuery;import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.FuzzyQuery;@@ -35,22 +34,22 @@ import org.apache.lucene.util.automaton.ByteRunAutomaton;* @lucene.internal*/final class MultiTermHighlighting {- private MultiTermHighlighting() {- }+ private MultiTermHighlighting() {}/**- * Extracts MultiTermQueries that match the provided field predicate.- * Returns equivalent automata that will match terms.+ * Extracts MultiTermQueries that match the provided field predicate. Returns equivalent automata+ * that will match terms.*/- static LabelledCharArrayMatcher[] extractAutomata(Query query, Predicate<String> fieldMatcher, boolean lookInSpan) {+ static LabelledCharArrayMatcher[] extractAutomata(+ Query query, Predicate<String> fieldMatcher, boolean lookInSpan) {AutomataCollector collector = new AutomataCollector(lookInSpan, fieldMatcher);query.visit(collector);return collector.runAutomata.toArray(new LabelledCharArrayMatcher[0]);}/**- * Indicates if the the leaf query (from {@link QueryVisitor#visitLeaf(Query)}) is a type of query that- * we can extract automata from.+ * Indicates if the the leaf query (from {@link QueryVisitor#visitLeaf(Query)}) is a type of query+ * that we can extract automata from.*/public static boolean canExtractAutomataFromLeafQuery(Query query) {return query instanceof AutomatonQuery || query instanceof FuzzyQuery;@@ -81,10 +80,9 @@ final class MultiTermHighlighting {}@Override- public void consumeTermsMatching(Query query, String field, Supplier<ByteRunAutomaton> automaton) {+ public void consumeTermsMatching(+ Query query, String field, Supplier<ByteRunAutomaton> automaton) {runAutomata.add(LabelledCharArrayMatcher.wrap(query.toString(), automaton.get()));}-}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/NoOpOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/NoOpOffsetStrategy.javaindex d69d1cb33c3..bfb5b593f47 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/NoOpOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/NoOpOffsetStrategy.java@@ -18,7 +18,6 @@ package org.apache.lucene.search.uhighlight;import java.io.IOException;import java.util.Collections;-import org.apache.lucene.index.LeafReader;import org.apache.lucene.search.MatchNoDocsQuery;import org.apache.lucene.util.BytesRef;@@ -33,8 +32,16 @@ public class NoOpOffsetStrategy extends FieldOffsetStrategy {public static final NoOpOffsetStrategy INSTANCE = new NoOpOffsetStrategy();private NoOpOffsetStrategy() {- super(new UHComponents("_ignored_", (s) -> false, new MatchNoDocsQuery(),- new BytesRef[0], PhraseHelper.NONE, new LabelledCharArrayMatcher[0], false, Collections.emptySet()));+ super(+ new UHComponents(+ "_ignored_",+ (s) -> false,+ new MatchNoDocsQuery(),+ new BytesRef[0],+ PhraseHelper.NONE,+ new LabelledCharArrayMatcher[0],+ false,+ Collections.emptySet()));}@Override@@ -43,8 +50,8 @@ public class NoOpOffsetStrategy extends FieldOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {return OffsetsEnum.EMPTY;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.javaindex d3998833f46..6743380e058 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OffsetsEnum.java@@ -24,7 +24,6 @@ import java.util.List;import java.util.Objects;import java.util.PriorityQueue;import java.util.function.Supplier;-import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.Term;import org.apache.lucene.search.MatchesIterator;@@ -35,9 +34,9 @@ import org.apache.lucene.util.BytesRefBuilder;import org.apache.lucene.util.IOUtils;/**- * An enumeration/iterator of a term and its offsets for use by {@link FieldHighlighter}.- * It is advanced and is placed in a priority queue by- * {@link FieldHighlighter#highlightOffsetsEnums(OffsetsEnum)} based on the start offset.+ * An enumeration/iterator of a term and its offsets for use by {@link FieldHighlighter}. It is+ * advanced and is placed in a priority queue by {@link+ * FieldHighlighter#highlightOffsetsEnums(OffsetsEnum)} based on the start offset.** @lucene.internal*/@@ -75,8 +74,8 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable}/**- * Advances to the next position and returns true, or if can't then returns false.- * Note that the initial state of this class is not positioned.+ * Advances to the next position and returns true, or if can't then returns false. Note that the+ * initial state of this class is not positioned.*/public abstract boolean nextPosition() throws IOException;@@ -84,8 +83,8 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeablepublic abstract int freq() throws IOException;/**- * The term at this position.- * This BytesRef is safe to continue to refer to, even after we move to the next position.+ * The term at this position. This BytesRef is safe to continue to refer to, even after we move to+ * the next position.** @see Passage#getMatchTerms()*/@@ -96,8 +95,7 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeablepublic abstract int endOffset() throws IOException;@Override- public void close() throws IOException {- }+ public void close() throws IOException {}@Overridepublic String toString() {@@ -106,7 +104,7 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeabletry {offset = ",[" + startOffset() + "-" + endOffset() + "]";} catch (Exception e) {- //ignore; for debugging only+ // ignore; for debugging only}try {return name + "(term:" + getTerm().utf8ToString() + offset + ")";@@ -115,9 +113,7 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable}}- /**- * Based on a {@link PostingsEnum} -- the typical/standard OE impl.- */+ /** Based on a {@link PostingsEnum} -- the typical/standard OE impl. */public static class OfPostings extends OffsetsEnum {private final BytesRef term;private final PostingsEnum postingsEnum; // with offsets@@ -174,12 +170,13 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable/** Based on a {@link MatchesIterator} with submatches. */public static class OfMatchesIteratorWithSubs extends OffsetsEnum {- //Either CachedOE impls (which are the submatches) or OfMatchesIterator impls+ // Either CachedOE impls (which are the submatches) or OfMatchesIterator implsprivate final PriorityQueue<OffsetsEnum> pendingQueue = new PriorityQueue<>();- private final HashMap<Query,BytesRef> queryToTermMap = new HashMap<>();+ private final HashMap<Query, BytesRef> queryToTermMap = new HashMap<>();public OfMatchesIteratorWithSubs(MatchesIterator matchesIterator) {- pendingQueue.add(new OfMatchesIterator(matchesIterator, () -> queryToTerm(matchesIterator.getQuery())));+ pendingQueue.add(+ new OfMatchesIterator(matchesIterator, () -> queryToTerm(matchesIterator.getQuery())));}@Override@@ -191,9 +188,9 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, CloseableOffsetsEnum newHeadOE = pendingQueue.peek();if (newHeadOE instanceof OfMatchesIterator) {// We found the matchesIterator. Requires processing.- nextWhenMatchesIterator((OfMatchesIterator) newHeadOE); // May or may not remove or re-queue itself+ // May or may not remove or re-queue itself+ nextWhenMatchesIterator((OfMatchesIterator) newHeadOE);} // else new head is a CacheOE or no more. Nothing to do with it.-} else { // formerHeadOE is OfMatchesIterator; advance itOfMatchesIterator miOE = (OfMatchesIterator) formerHeadOE;if (miOE.nextPosition()) {@@ -233,38 +230,48 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeablewhile (thisMI.next()) {if (false == enqueueCachedMatches(thisMI.getSubMatches())) { // recursion// if no sub-matches then add ourselves- pendingQueue.add(new CachedOE(queryToTerm(thisMI.getQuery()), thisMI.startOffset(), thisMI.endOffset()));+ pendingQueue.add(+ new CachedOE(+ queryToTerm(thisMI.getQuery()), thisMI.startOffset(), thisMI.endOffset()));}}return true;}}- /** Maps a Query from {@link MatchesIterator#getQuery()} to {@link OffsetsEnum#getTerm()}.- * See {@link Passage#getMatchTerms()}. */+ /**+ * Maps a Query from {@link MatchesIterator#getQuery()} to {@link OffsetsEnum#getTerm()}. See+ * {@link Passage#getMatchTerms()}.+ */private BytesRef queryToTerm(Query query) {- // compute an approximate BytesRef term of a Query. We cache this since we're likely to see the same query again.- // Our approach is to visit each matching term in order, concatenating them with an adjoining space.- // If we don't have any (perhaps due to an MTQ like a wildcard) then we fall back on the toString() of the query.- return queryToTermMap.computeIfAbsent(query, (Query q) -> {- BytesRefBuilder bytesRefBuilder = new BytesRefBuilder();- q.visit(new QueryVisitor() {- @Override- public void consumeTerms(Query query, Term... terms) {- for (Term term : terms) {- if (bytesRefBuilder.length() > 0) {- bytesRefBuilder.append((byte) ' ');- }- bytesRefBuilder.append(term.bytes());+ // compute an approximate BytesRef term of a Query. We cache this since we're likely to see+ // the same query again.+ // Our approach is to visit each matching term in order, concatenating them with an adjoining+ // space.+ // If we don't have any (perhaps due to an MTQ like a wildcard) then we fall back on the+ // toString() of the query.+ return queryToTermMap.computeIfAbsent(+ query,+ (Query q) -> {+ BytesRefBuilder bytesRefBuilder = new BytesRefBuilder();+ q.visit(+ new QueryVisitor() {+ @Override+ public void consumeTerms(Query query, Term... terms) {+ for (Term term : terms) {+ if (bytesRefBuilder.length() > 0) {+ bytesRefBuilder.append((byte) ' ');+ }+ bytesRefBuilder.append(term.bytes());+ }+ }+ });+ if (bytesRefBuilder.length() > 0) {+ return bytesRefBuilder.get();}- }- });- if (bytesRefBuilder.length() > 0) {- return bytesRefBuilder.get();- }- // fallback: (likely a MultiTermQuery)- return new BytesRef(q.toString());- });+ // fallback: (likely a MultiTermQuery)+ return new BytesRef(q.toString());+ });}@Override@@ -361,41 +368,38 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeable}}- /**- * Empty enumeration- */- public static final OffsetsEnum EMPTY = new OffsetsEnum() {- @Override- public boolean nextPosition() throws IOException {- return false;- }-- @Override- public BytesRef getTerm() throws IOException {- throw new UnsupportedOperationException();- }+ /** Empty enumeration */+ public static final OffsetsEnum EMPTY =+ new OffsetsEnum() {+ @Override+ public boolean nextPosition() throws IOException {+ return false;+ }- @Override- public int startOffset() throws IOException {- throw new UnsupportedOperationException();- }+ @Override+ public BytesRef getTerm() throws IOException {+ throw new UnsupportedOperationException();+ }- @Override- public int endOffset() throws IOException {- throw new UnsupportedOperationException();- }+ @Override+ public int startOffset() throws IOException {+ throw new UnsupportedOperationException();+ }- @Override- public int freq() throws IOException {- return 0;- }+ @Override+ public int endOffset() throws IOException {+ throw new UnsupportedOperationException();+ }- };+ @Override+ public int freq() throws IOException {+ return 0;+ }+ };- /**- * A view over several OffsetsEnum instances, merging them in-place- */- //If OffsetsEnum and MatchesIterator ever truly merge then this could go away in lieu of DisjunctionMatchesIterator+ /** A view over several OffsetsEnum instances, merging them in-place */+ // If OffsetsEnum and MatchesIterator ever truly merge then this could go away in lieu of+ // DisjunctionMatchesIteratorpublic static class MultiOffsetsEnum extends OffsetsEnum {private final PriorityQueue<OffsetsEnum> queue;@@ -404,8 +408,7 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeablepublic MultiOffsetsEnum(List<OffsetsEnum> inner) throws IOException {this.queue = new PriorityQueue<>();for (OffsetsEnum oe : inner) {- if (oe.nextPosition())- this.queue.add(oe);+ if (oe.nextPosition()) this.queue.add(oe);}}@@ -420,8 +423,7 @@ public abstract class OffsetsEnum implements Comparable<OffsetsEnum>, Closeableif (top.nextPosition()) {this.queue.add(top);return true;- }- else {+ } else {top.close();}return this.queue.size() > 0;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OverlaySingleDocTermsLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OverlaySingleDocTermsLeafReader.javaindex 53f29e348ca..74425eb8c3d 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OverlaySingleDocTermsLeafReader.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/OverlaySingleDocTermsLeafReader.java@@ -18,7 +18,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.PostingsEnum;@@ -28,8 +27,8 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.automaton.CompiledAutomaton;/**- * Overlays a 2nd LeafReader for the terms of one field, otherwise the primary reader is- * consulted. The 2nd reader is assumed to have one document of 0 and we remap it to a target doc ID.+ * Overlays a 2nd LeafReader for the terms of one field, otherwise the primary reader is consulted.+ * The 2nd reader is assumed to have one document of 0 and we remap it to a target doc ID.** @lucene.internal*/@@ -39,7 +38,8 @@ public class OverlaySingleDocTermsLeafReader extends FilterLeafReader {private final String in2Field;private final int in2TargetDocId;- public OverlaySingleDocTermsLeafReader(LeafReader in, LeafReader in2, String in2Field, int in2TargetDocId) {+ public OverlaySingleDocTermsLeafReader(+ LeafReader in, LeafReader in2, String in2Field, int in2TargetDocId) {super(in);this.in2 = in2;this.in2Field = in2Field;@@ -68,7 +68,8 @@ public class OverlaySingleDocTermsLeafReader extends FilterLeafReader {}@Override- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {+ public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm)+ throws IOException {return filterTermsEnum(super.intersect(compiled, startTerm));}@@ -76,7 +77,7 @@ public class OverlaySingleDocTermsLeafReader extends FilterLeafReader {return new FilterTermsEnum(termsEnum) {@Overridepublic PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {- //TODO 'reuse' will always fail to reuse unless we unwrap it+ // TODO 'reuse' will always fail to reuse unless we unwrap itreturn new FilterPostingsEnum(super.postings(reuse, flags)) {@Overridepublic int nextDoc() throws IOException {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.javaindex 1c760cba994..562991a8e20 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/Passage.java@@ -16,16 +16,15 @@*/package org.apache.lucene.search.uhighlight;-import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.RamUsageEstimator;/*** Represents a passage (typically a sentence of the document).- * <p>- * A passage contains {@link #getNumMatches} highlights from the query,- * and the offsets and query terms that correspond with each match.+ *+ * <p>A passage contains {@link #getNumMatches} highlights from the query, and the offsets and query+ * terms that correspond with each match.** @lucene.experimental*/@@ -73,7 +72,7 @@ public class Passage {numMatches = 0;}- /** For debugging. ex: Passage[0-22]{yin[0-3],yang[4-8],yin[10-13]}score=2.4964213 */+ /** For debugging. ex: Passage[0-22]{yin[0-3],yang[4-8],yin[10-13]}score=2.4964213 */@Overridepublic String toString() {StringBuilder buf = new StringBuilder();@@ -84,7 +83,11 @@ public class Passage {buf.append(',');}buf.append(matchTerms[i].utf8ToString());- buf.append('[').append(matchStarts[i] - startOffset).append('-').append(matchEnds[i] - startOffset).append(']');+ buf.append('[')+ .append(matchStarts[i] - startOffset)+ .append('-')+ .append(matchEnds[i] - startOffset)+ .append(']');}buf.append('}');buf.append("score=").append(score);@@ -94,8 +97,7 @@ public class Passage {/*** Start offset of this passage.*- * @return start index (inclusive) of the passage in the- * original content: always >= 0.+ * @return start index (inclusive) of the passage in the original content: always >= 0.*/public int getStartOffset() {return startOffset;@@ -104,8 +106,8 @@ public class Passage {/*** End offset of this passage.*- * @return end index (exclusive) of the passage in the- * original content: always >= {@link #getStartOffset()}+ * @return end index (exclusive) of the passage in the original content: always >= {@link+ * #getStartOffset()}*/public int getEndOffset() {return endOffset;@@ -115,9 +117,7 @@ public class Passage {return endOffset - startOffset;}- /**- * Passage's score.- */+ /** Passage's score. */public float getScore() {return score;}@@ -127,9 +127,8 @@ public class Passage {}/**- * Number of term matches available in- * {@link #getMatchStarts}, {@link #getMatchEnds},- * {@link #getMatchTerms}+ * Number of term matches available in {@link #getMatchStarts}, {@link #getMatchEnds}, {@link+ * #getMatchTerms}*/public int getNumMatches() {return numMatches;@@ -137,9 +136,9 @@ public class Passage {/*** Start offsets of the term matches, in increasing order.- * <p>- * Only {@link #getNumMatches} are valid. Note that these- * offsets are absolute (not relative to {@link #getStartOffset()}).+ *+ * <p>Only {@link #getNumMatches} are valid. Note that these offsets are absolute (not relative to+ * {@link #getStartOffset()}).*/public int[] getMatchStarts() {return matchStarts;@@ -147,28 +146,28 @@ public class Passage {/*** End offsets of the term matches, corresponding with {@link #getMatchStarts}.- * <p>- * Only {@link #getNumMatches} are valid. Note that its possible that an end offset- * could exceed beyond the bounds of the passage ({@link #getEndOffset()}), if the- * Analyzer produced a term which spans a passage boundary.+ *+ * <p>Only {@link #getNumMatches} are valid. Note that its possible that an end offset could+ * exceed beyond the bounds of the passage ({@link #getEndOffset()}), if the Analyzer produced a+ * term which spans a passage boundary.*/public int[] getMatchEnds() {return matchEnds;}/**- * BytesRef (term text) of the matches, corresponding with {@link #getMatchStarts()}. The primary purpose of this- * method is to expose the number of unique terms per passage for use in passage scoring.- * The actual term byte content is not well defined by this highlighter, and thus use of it is more subject to- * change.- * <p>- * The term might be simply the analyzed term at this position.- * Depending on the highlighter's configuration, the match term may be a phrase (instead of a word), and in such- * a case might be a series of space-separated analyzed terms.- * If the match is from a {@link org.apache.lucene.search.MultiTermQuery} then the match term may be the toString() of- * that query.- * <p>- * Only {@link #getNumMatches()} are valid.+ * BytesRef (term text) of the matches, corresponding with {@link #getMatchStarts()}. The primary+ * purpose of this method is to expose the number of unique terms per passage for use in passage+ * scoring. The actual term byte content is not well defined by this highlighter, and thus use of+ * it is more subject to change.+ *+ * <p>The term might be simply the analyzed term at this position. Depending on the highlighter's+ * configuration, the match term may be a phrase (instead of a word), and in such a case might be+ * a series of space-separated analyzed terms. If the match is from a {@link+ * org.apache.lucene.search.MultiTermQuery} then the match term may be the toString() of that+ * query.+ *+ * <p>Only {@link #getNumMatches()} are valid.*/public BytesRef[] getMatchTerms() {return matchTerms;@@ -188,5 +187,4 @@ public class Passage {assert startOffset <= endOffset;this.endOffset = endOffset;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageFormatter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageFormatter.javaindex c2de2aae345..135002b5d78 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageFormatter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageFormatter.java@@ -24,17 +24,15 @@ package org.apache.lucene.search.uhighlight;public abstract class PassageFormatter {/**- * Formats the top <code>passages</code> from <code>content</code>- * into a human-readable text snippet.+ * Formats the top <code>passages</code> from <code>content</code> into a human-readable text+ * snippet.*- * @param passages top-N passages for the field. Note these are sorted in- * the order that they appear in the document for convenience.- * @param content content for the field.- * @return formatted highlight. Note that for the- * non-expert APIs in {@link UnifiedHighlighter} that- * return String, the toString method on the Object- * returned by this method is used to compute the string.+ * @param passages top-N passages for the field. Note these are sorted in the order that they+ * appear in the document for convenience.+ * @param content content for the field.+ * @return formatted highlight. Note that for the non-expert APIs in {@link UnifiedHighlighter}+ * that return String, the toString method on the Object returned by this method is used to+ * compute the string.*/public abstract Object format(Passage passages[], String content);-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageScorer.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageScorer.javaindex 4a7a00d8441..34b5b662d3e 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageScorer.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PassageScorer.java@@ -17,15 +17,14 @@package org.apache.lucene.search.uhighlight;import java.util.Arrays;-import org.apache.lucene.util.BytesRefHash;/*** Ranks passages found by {@link UnifiedHighlighter}.- * <p>- * Each passage is scored as a miniature document within the document.- * The final score is computed as {@link #norm} * ∑ ({@link #weight} * {@link #tf}).- * The default implementation is {@link #norm} * BM25.+ *+ * <p>Each passage is scored as a miniature document within the document. The final score is+ * computed as {@link #norm} * ∑ ({@link #weight} * {@link #tf}). The default implementation is+ * {@link #norm} * BM25.** @lucene.experimental*/@@ -33,25 +32,20 @@ public class PassageScorer {// TODO: this formula is completely made up. It might not provide relevant snippets!- /**- * BM25 k1 parameter, controls term frequency normalization- */+ /** BM25 k1 parameter, controls term frequency normalization */final float k1;- /**- * BM25 b parameter, controls length normalization.- */+ /** BM25 b parameter, controls length normalization. */final float b;- /**- * A pivot used for length normalization.- */+ /** A pivot used for length normalization. */final float pivot;/*** Creates PassageScorer with these default values:+ ** <ul>- * <li>{@code k1 = 1.2},- * <li>{@code b = 0.75}.- * <li>{@code pivot = 87}+ * <li>{@code k1 = 1.2},+ * <li>{@code b = 0.75}.+ * <li>{@code pivot = 87}* </ul>*/public PassageScorer() {@@ -63,9 +57,10 @@ public class PassageScorer {/*** Creates PassageScorer with specified scoring parameters*- * @param k1 Controls non-linear term frequency normalization (saturation).- * @param b Controls to what degree passage length normalizes tf values.- * @param pivot Pivot value for length normalization (some rough idea of average sentence length in characters).+ * @param k1 Controls non-linear term frequency normalization (saturation).+ * @param b Controls to what degree passage length normalizes tf values.+ * @param pivot Pivot value for length normalization (some rough idea of average sentence length+ * in characters).*/public PassageScorer(float k1, float b, float pivot) {this.k1 = k1;@@ -88,10 +83,9 @@ public class PassageScorer {}/**- * Computes term weight, given the frequency within the passage- * and the passage's length.+ * Computes term weight, given the frequency within the passage and the passage's length.*- * @param freq number of occurrences of within this passage+ * @param freq number of occurrences of within this passage* @param passageLen length of the passage in characters.* @return term weight*/@@ -102,11 +96,11 @@ public class PassageScorer {/*** Normalize a passage according to its position in the document.- * <p>- * Typically passages towards the beginning of the document are- * more useful for summarizing the contents.- * <p>- * The default implementation is <code>1 + 1/log(pivot + passageStart)</code>+ *+ * <p>Typically passages towards the beginning of the document are more useful for summarizing the+ * contents.+ *+ * <p>The default implementation is <code>1 + 1/log(pivot + passageStart)</code>** @param passageStart start offset of the passage* @return a boost value multiplied into the passage's core.@@ -127,15 +121,15 @@ public class PassageScorer {int termIndex = termsHash.add(passage.getMatchTerms()[i]);if (termIndex < 0) {termIndex = -(termIndex + 1);- }- else {+ } else {termFreqsInDoc[termIndex] = passage.getMatchTermFreqsInDoc()[i];}termFreqsInPassage[termIndex]++;}for (int i = 0; i < termsHash.size(); i++) {- score += tf(termFreqsInPassage[i], passage.getLength()) * weight(contentLength, termFreqsInDoc[i]);+ score +=+ tf(termFreqsInPassage[i], passage.getLength()) * weight(contentLength, termFreqsInDoc[i]);}score *= norm(passage.getStartOffset());return score;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.javaindex 47b770dfcac..96c8e88b247 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PhraseHelper.java@@ -26,7 +26,6 @@ import java.util.Map;import java.util.Set;import java.util.function.Function;import java.util.function.Predicate;-import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.LeafReader;@@ -55,17 +54,23 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.PriorityQueue;/**- * Helps the {@link FieldOffsetStrategy} with position sensitive queries (e.g. highlight phrases correctly).- * This is a stateful class holding information about the query, but it can (and is) re-used across highlighting- * documents. Despite this state, it's immutable after construction.+ * Helps the {@link FieldOffsetStrategy} with position sensitive queries (e.g. highlight phrases+ * correctly). This is a stateful class holding information about the query, but it can (and is)+ * re-used across highlighting documents. Despite this state, it's immutable after construction.** @lucene.internal*/// TODO rename to SpanHighlighting ?public class PhraseHelper {- public static final PhraseHelper NONE = new PhraseHelper(new MatchAllDocsQuery(), "_ignored_",- (s) -> false, spanQuery -> null, query -> null, true);+ public static final PhraseHelper NONE =+ new PhraseHelper(+ new MatchAllDocsQuery(),+ "_ignored_",+ (s) -> false,+ spanQuery -> null,+ query -> null,+ true);private final String fieldName;private final Set<BytesRef> positionInsensitiveTerms; // (TermQuery terms)@@ -74,19 +79,23 @@ public class PhraseHelper {private final Predicate<String> fieldMatcher;/**- * Constructor.- * {@code rewriteQueryPred} is an extension hook to override the default choice of- * {@link WeightedSpanTermExtractor#mustRewriteQuery(SpanQuery)}. By default unknown query types are rewritten,- * so use this to return {@link Boolean#FALSE} if you know the query doesn't need to be rewritten.- * Similarly, {@code preExtractRewriteFunction} is also an extension hook for extract to allow different queries- * to be set before the {@link WeightedSpanTermExtractor}'s extraction is invoked.- * {@code ignoreQueriesNeedingRewrite} effectively ignores any query clause that needs to be "rewritten", which is- * usually limited to just a {@link SpanMultiTermQueryWrapper} but could be other custom ones.- * {@code fieldMatcher} The field name predicate to use for extracting the query part that must be highlighted.+ * Constructor. {@code rewriteQueryPred} is an extension hook to override the default choice of+ * {@link WeightedSpanTermExtractor#mustRewriteQuery(SpanQuery)}. By default unknown query types+ * are rewritten, so use this to return {@link Boolean#FALSE} if you know the query doesn't need+ * to be rewritten. Similarly, {@code preExtractRewriteFunction} is also an extension hook for+ * extract to allow different queries to be set before the {@link WeightedSpanTermExtractor}'s+ * extraction is invoked. {@code ignoreQueriesNeedingRewrite} effectively ignores any query clause+ * that needs to be "rewritten", which is usually limited to just a {@link+ * SpanMultiTermQueryWrapper} but could be other custom ones. {@code fieldMatcher} The field name+ * predicate to use for extracting the query part that must be highlighted.*/- public PhraseHelper(Query query, String field, Predicate<String> fieldMatcher, Function<SpanQuery, Boolean> rewriteQueryPred,- Function<Query, Collection<Query>> preExtractRewriteFunction,- boolean ignoreQueriesNeedingRewrite) {+ public PhraseHelper(+ Query query,+ String field,+ Predicate<String> fieldMatcher,+ Function<SpanQuery, Boolean> rewriteQueryPred,+ Function<Query, Collection<Query>> preExtractRewriteFunction,+ boolean ignoreQueriesNeedingRewrite) {this.fieldName = field;this.fieldMatcher = fieldMatcher;// filter terms to those we want@@ -95,24 +104,28 @@ public class PhraseHelper {// TODO Have toSpanQuery(query) Function as an extension point for those with custom Query impls- boolean[] mustRewriteHolder = {false}; // boolean wrapped in 1-ary array so it's mutable from inner class+ // boolean wrapped in 1-ary array so it's mutable from inner class+ boolean[] mustRewriteHolder = {false};// For TermQueries or other position insensitive queries, collect the Terms.- // For other Query types, WSTE will convert to an equivalent SpanQuery. NOT extracting position spans here.+ // For other Query types, WSTE will convert to an equivalent SpanQuery. NOT extracting position+ // spans here.new WeightedSpanTermExtractor(field) {- //anonymous constructor+ // anonymous constructor{- setExpandMultiTermQuery(true); //necessary for mustRewriteQuery(spanQuery) to work.+ setExpandMultiTermQuery(true); // necessary for mustRewriteQuery(spanQuery) to work.try {- extract(query, 1f, null); // null because we won't actually extract right now; we're not collecting+ // null because we won't actually extract right now; we're not collecting+ extract(query, 1f, null);} catch (Exception e) {throw new RuntimeException(e);}}@Override- protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms) throws IOException {+ protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms)+ throws IOException {Collection<Query> newQueriesToExtract = preExtractRewriteFunction.apply(query);if (newQueriesToExtract != null) {for (Query newQuery : newQueriesToExtract) {@@ -126,34 +139,39 @@ public class PhraseHelper {@Overrideprotected boolean isQueryUnsupported(Class<? extends Query> clazz) {if (clazz.isAssignableFrom(MultiTermQuery.class)) {- return true; //We do MTQ processing separately in MultiTermHighlighting.java+ return true; // We do MTQ processing separately in MultiTermHighlighting.java}- return true; //TODO set to false and provide a hook to customize certain queries.+ return true; // TODO set to false and provide a hook to customize certain queries.}// called on Query types that are NOT position sensitive, e.g. TermQuery@Override- protected void extractWeightedTerms(Map<String, WeightedSpanTerm> terms, Query query, float boost) {- query.visit(new QueryVisitor() {- @Override- public boolean acceptField(String field) {- return fieldMatcher.test(field);- }- @Override- public void consumeTerms(Query query, Term... terms) {- for (Term term : terms) {- positionInsensitiveTerms.add(term.bytes());- }- }- });+ protected void extractWeightedTerms(+ Map<String, WeightedSpanTerm> terms, Query query, float boost) {+ query.visit(+ new QueryVisitor() {+ @Override+ public boolean acceptField(String field) {+ return fieldMatcher.test(field);+ }++ @Override+ public void consumeTerms(Query query, Term... terms) {+ for (Term term : terms) {+ positionInsensitiveTerms.add(term.bytes());+ }+ }+ });}- // called on SpanQueries. Some other position-sensitive queries like PhraseQuery are converted beforehand+ // called on SpanQueries. Some other position-sensitive queries like PhraseQuery are converted+ // beforehand@Override- protected void extractWeightedSpanTerms(Map<String, WeightedSpanTerm> terms, SpanQuery spanQuery,- float boost) throws IOException {+ protected void extractWeightedSpanTerms(+ Map<String, WeightedSpanTerm> terms, SpanQuery spanQuery, float boost)+ throws IOException {// if this span query isn't for this field, skip it.- Set<String> fieldNameSet = new HashSet<>();//TODO reuse. note: almost always size 1+ Set<String> fieldNameSet = new HashSet<>(); // TODO reuse. note: almost always size 1collectSpanQueryFields(spanQuery, fieldNameSet);for (String spanField : fieldNameSet) {if (!fieldMatcher.test(spanField)) {@@ -163,7 +181,7 @@ public class PhraseHelper {boolean mustRewriteQuery = mustRewriteQuery(spanQuery);if (ignoreQueriesNeedingRewrite && mustRewriteQuery) {- return;// ignore this query+ return; // ignore this query}mustRewriteHolder[0] |= mustRewriteQuery;@@ -172,7 +190,7 @@ public class PhraseHelper {@Overrideprotected boolean mustRewriteQuery(SpanQuery spanQuery) {- Boolean rewriteQ = rewriteQueryPred.apply(spanQuery);// allow to override+ Boolean rewriteQ = rewriteQueryPred.apply(spanQuery); // allow to overridereturn rewriteQ != null ? rewriteQ : super.mustRewriteQuery(spanQuery);}}; // calling the constructor triggered the extraction/visiting we want. Hacky; yes.@@ -184,17 +202,15 @@ public class PhraseHelper {return spanQueries;}- /**- * If there is no position sensitivity then use of the instance of this class can be ignored.- */+ /** If there is no position sensitivity then use of the instance of this class can be ignored. */public boolean hasPositionSensitivity() {return spanQueries.isEmpty() == false;}/*** Rewrite is needed for handling a {@link SpanMultiTermQueryWrapper} (MTQ / wildcards) or some- * custom things. When true, the resulting term list will probably be different than what it was known- * to be initially.+ * custom things. When true, the resulting term list will probably be different than what it was+ * known to be initially.*/public boolean willRewrite() {return willRewrite;@@ -202,37 +218,45 @@ public class PhraseHelper {/** Returns the terms that are position-insensitive (sorted). */public BytesRef[] getAllPositionInsensitiveTerms() {- BytesRef[] result = positionInsensitiveTerms.toArray(new BytesRef[positionInsensitiveTerms.size()]);+ BytesRef[] result =+ positionInsensitiveTerms.toArray(new BytesRef[positionInsensitiveTerms.size()]);Arrays.sort(result);return result;}- /** Given the internal SpanQueries, produce a number of OffsetsEnum into the {@code results} param. */- public void createOffsetsEnumsForSpans(LeafReader leafReader, int docId, List<OffsetsEnum> results) throws IOException {+ /**+ * Given the internal SpanQueries, produce a number of OffsetsEnum into the {@code results} param.+ */+ public void createOffsetsEnumsForSpans(+ LeafReader leafReader, int docId, List<OffsetsEnum> results) throws IOException {leafReader = new SingleFieldWithOffsetsFilterLeafReader(leafReader, fieldName);- //TODO avoid searcher and do what it does to rewrite & get weight?+ // TODO avoid searcher and do what it does to rewrite & get weight?IndexSearcher searcher = new IndexSearcher(leafReader);searcher.setQueryCache(null);// for each SpanQuery, grab it's Spans and put it into a PriorityQueue- PriorityQueue<Spans> spansPriorityQueue = new PriorityQueue<Spans>(spanQueries.size()) {- @Override- protected boolean lessThan(Spans a, Spans b) {- return a.startPosition() <= b.startPosition();- }- };+ PriorityQueue<Spans> spansPriorityQueue =+ new PriorityQueue<Spans>(spanQueries.size()) {+ @Override+ protected boolean lessThan(Spans a, Spans b) {+ return a.startPosition() <= b.startPosition();+ }+ };for (Query query : spanQueries) {- Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1);+ Weight weight =+ searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1);Scorer scorer = weight.scorer(leafReader.getContext());if (scorer == null) {continue;}TwoPhaseIterator twoPhaseIterator = scorer.twoPhaseIterator();if (twoPhaseIterator != null) {- if (twoPhaseIterator.approximation().advance(docId) != docId || !twoPhaseIterator.matches()) {+ if (twoPhaseIterator.approximation().advance(docId) != docId+ || !twoPhaseIterator.matches()) {continue;}- } else if (scorer.iterator().advance(docId) != docId) { // preposition, and return doing nothing if find none+ } else if (scorer.iterator().advance(docId)+ != docId) { // preposition, and return doing nothing if find nonecontinue;}@@ -243,17 +267,21 @@ public class PhraseHelper {}}- // Iterate the Spans in the PriorityQueue, collecting as we go. By using a PriorityQueue ordered by position,- // the underlying offsets in our collector will be mostly appended to the end of arrays (efficient).+ // Iterate the Spans in the PriorityQueue, collecting as we go. By using a PriorityQueue+ // ordered by position,+ // the underlying offsets in our collector will be mostly appended to the end of arrays+ // (efficient).// note: alternatively it'd interesting if we produced one OffsetsEnum that internally advanced- // this PriorityQueue when nextPosition is called; it would cap what we have to cache for large docs and+ // this PriorityQueue when nextPosition is called; it would cap what we have to cache for+ // large docs and// exiting early (due to maxLen) is easy.- // But at least we have an accurate "freq" and it shouldn't be too much data to collect. Even SpanScorer+ // But at least we have an accurate "freq" and it shouldn't be too much data to collect. Even+ // SpanScorer// navigates the spans fully to compute a good freq (and thus score)!OffsetSpanCollector spanCollector = new OffsetSpanCollector();while (spansPriorityQueue.size() > 0) {Spans spans = spansPriorityQueue.top();- //TODO limit to a capped endOffset length somehow so we can break this loop early+ // TODO limit to a capped endOffset length somehow so we can break this loop earlyspans.collect(spanCollector);if (spans.nextStartPosition() == Spans.NO_MORE_POSITIONS) {@@ -267,9 +295,9 @@ public class PhraseHelper {/*** Needed to support the ability to highlight a query irrespective of the field a query refers to- * (aka requireFieldMatch=false).- * This reader will just delegate every call to a single field in the wrapped- * LeafReader. This way we ensure that all queries going through this reader target the same field.+ * (aka requireFieldMatch=false). This reader will just delegate every call to a single field in+ * the wrapped LeafReader. This way we ensure that all queries going through this reader target+ * the same field.*/private static final class SingleFieldWithOffsetsFilterLeafReader extends FilterLeafReader {final String fieldName;@@ -281,12 +309,13 @@ public class PhraseHelper {@Overridepublic FieldInfos getFieldInfos() {- throw new UnsupportedOperationException();//TODO merge them+ throw new UnsupportedOperationException(); // TODO merge them}@Overridepublic Terms terms(String field) throws IOException {- // ensure the underlying PostingsEnum returns offsets. It's sad we have to do this to use the SpanCollector.+ // ensure the underlying PostingsEnum returns offsets. It's sad we have to do this to use the+ // SpanCollector.return new FilterTerms(super.terms(fieldName)) {@Overridepublic TermsEnum iterator() throws IOException {@@ -327,7 +356,8 @@ public class PhraseHelper {SpanCollectedOffsetsEnum offsetsEnum = termToOffsetsEnums.get(term.bytes());if (offsetsEnum == null) {- // If it's pos insensitive we handle it outside of PhraseHelper. term.field() is from the Query.+ // If it's pos insensitive we handle it outside of PhraseHelper. term.field() is from the+ // Query.if (positionInsensitiveTerms.contains(term.bytes())) {return;}@@ -352,7 +382,8 @@ public class PhraseHelper {private SpanCollectedOffsetsEnum(BytesRef term, int postingsFreq) {this.term = term;- this.startOffsets = new int[postingsFreq]; // hopefully not wasteful? At least we needn't resize it.+ // hopefully not wasteful? At least we needn't resize it.+ this.startOffsets = new int[postingsFreq];this.endOffsets = new int[postingsFreq];}@@ -372,7 +403,7 @@ public class PhraseHelper {if (cmp == 0) {return; // we already have this offset-pair for this term} else if (cmp < 0) {- break; //we will insert offsetPair to the right of pairIdx+ break; // we will insert offsetPair to the right of pairIdx}}// pairIdx is now one position to the left of where we insert the new pair@@ -413,5 +444,4 @@ public class PhraseHelper {return endOffsets[enumIdx];}}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsOffsetStrategy.javaindex 53c1c95b4b6..535133df835 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsOffsetStrategy.java@@ -17,12 +17,11 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.LeafReader;/**- * Uses offsets in postings -- {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}. This+ * Uses offsets in postings -- {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}. This* does not support multi-term queries; the highlighter will fallback on analysis for that.** @lucene.internal@@ -34,11 +33,11 @@ public class PostingsOffsetStrategy extends FieldOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {return createOffsetsEnumFromReader(reader, docId);}-@Overridepublic UnifiedHighlighter.OffsetSource getOffsetSource() {return UnifiedHighlighter.OffsetSource.POSTINGS;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsWithTermVectorsOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsWithTermVectorsOffsetStrategy.javaindex b0249fa56a3..ae9405b5ff3 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsWithTermVectorsOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/PostingsWithTermVectorsOffsetStrategy.java@@ -17,12 +17,12 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.Terms;/**- * Like {@link PostingsOffsetStrategy} but also uses term vectors (only terms needed) for multi-term queries.+ * Like {@link PostingsOffsetStrategy} but also uses term vectors (only terms needed) for multi-term+ * queries.** @lucene.internal*/@@ -33,7 +33,8 @@ public class PostingsWithTermVectorsOffsetStrategy extends FieldOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader leafReader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader leafReader, int docId, String content)+ throws IOException {Terms docTerms = leafReader.getTermVector(docId, getField());if (docTerms == null) {return OffsetsEnum.EMPTY;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/SplittingBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/SplittingBreakIterator.javaindex b3a415c0caf..b13bb6126ba 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/SplittingBreakIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/SplittingBreakIterator.java@@ -21,12 +21,11 @@ import java.text.CharacterIterator;import java.text.StringCharacterIterator;/**- * Virtually slices the text on both sides of every occurrence of the specified character. If the slice is 0-length- * which happens for adjacent slice characters or when they are at the beginning or end, that character is reported- * as a boundary.- * For every slice between the specified characters, it is further processed with a specified- * BreakIterator. A consequence is that the enclosed BreakIterator will never "see" the splitting character.- * <br>+ * Virtually slices the text on both sides of every occurrence of the specified character. If the+ * slice is 0-length which happens for adjacent slice characters or when they are at the beginning+ * or end, that character is reported as a boundary. For every slice between the specified+ * characters, it is further processed with a specified BreakIterator. A consequence is that the+ * enclosed BreakIterator will never "see" the splitting character. <br>* <em>Note: {@link #setText(CharacterIterator)} is unsupported. Use the string version.</em>** @lucene.experimental@@ -81,7 +80,8 @@ public class SplittingBreakIterator extends BreakIterator {return current = sliceStartIdx;}baseIter.setText(text.substring(sliceStartIdx, sliceEndIdx));- return current = sliceStartIdx + baseIter.current();// since setText() sets to first(), just grab current()+ // since setText() sets to first(), just grab current()+ return current = sliceStartIdx + baseIter.current();}@Override@@ -91,7 +91,7 @@ public class SplittingBreakIterator extends BreakIterator {if (sliceStartIdx == -1) {sliceStartIdx = 0;} else {- sliceStartIdx++;//past sliceChar+ sliceStartIdx++; // past sliceChar}if (sliceEndIdx == sliceStartIdx) {return current = sliceEndIdx;@@ -108,7 +108,7 @@ public class SplittingBreakIterator extends BreakIterator {return current = current + sliceStartIdx;}if (sliceEndIdx >= text.length()) {- current = prevCurrent;//keep current where it is+ current = prevCurrent; // keep current where it isreturn DONE;}sliceStartIdx = sliceEndIdx + 1;@@ -120,7 +120,7 @@ public class SplittingBreakIterator extends BreakIterator {return current = sliceStartIdx;}baseIter.setText(text.substring(sliceStartIdx, sliceEndIdx));- return current = sliceStartIdx + baseIter.current();//use current() since at first() already+ return current = sliceStartIdx + baseIter.current(); // use current() since at first() already}@Override@@ -131,7 +131,7 @@ public class SplittingBreakIterator extends BreakIterator {return current = current + sliceStartIdx;}if (sliceStartIdx == 0) {- current = prevCurrent;//keep current where it is+ current = prevCurrent; // keep current where it isreturn DONE;}sliceEndIdx = sliceStartIdx - 1;@@ -139,7 +139,7 @@ public class SplittingBreakIterator extends BreakIterator {if (sliceStartIdx == -1) {sliceStartIdx = 0;} else {- sliceStartIdx++;//past sliceChar+ sliceStartIdx++; // past sliceChar}if (sliceStartIdx == sliceEndIdx) {return current = sliceStartIdx;@@ -156,17 +156,17 @@ public class SplittingBreakIterator extends BreakIterator {last(); // because https://bugs.openjdk.java.net/browse/JDK-8015110return DONE;}- sliceStartIdx = text.lastIndexOf(sliceChar, offset);//no +1+ sliceStartIdx = text.lastIndexOf(sliceChar, offset); // no +1if (sliceStartIdx == -1) {sliceStartIdx = 0;} else {- sliceStartIdx++;//move past separator+ sliceStartIdx++; // move past separator}sliceEndIdx = text.indexOf(sliceChar, Math.max(offset + 1, sliceStartIdx));if (sliceEndIdx == -1) {sliceEndIdx = text.length();}- if (sliceStartIdx != sliceEndIdx) {//otherwise, adjacent separator or separator at end+ if (sliceStartIdx != sliceEndIdx) { // otherwise, adjacent separator or separator at endbaseIter.setText(text.substring(sliceStartIdx, sliceEndIdx));}}@@ -175,7 +175,8 @@ public class SplittingBreakIterator extends BreakIterator {if (sliceStartIdx == sliceEndIdx) {return current = offset + 1;} else {- // note: following() can never be first() if the first character is a boundary (it usually is).+ // note: following() can never be first() if the first character is a boundary (it usually+ // is).// So we have to check if we should call first() instead of following():if (offset == sliceStartIdx - 1) {// the first boundary following this offset is the very first boundary in this slice@@ -193,7 +194,7 @@ public class SplittingBreakIterator extends BreakIterator {first(); // because https://bugs.openjdk.java.net/browse/JDK-8015110return DONE;}- sliceEndIdx = text.indexOf(sliceChar, offset);//no -1+ sliceEndIdx = text.indexOf(sliceChar, offset); // no -1if (sliceEndIdx == -1) {sliceEndIdx = text.length();}@@ -203,7 +204,7 @@ public class SplittingBreakIterator extends BreakIterator {} else {sliceStartIdx = Math.min(sliceStartIdx + 1, sliceEndIdx);}- if (sliceStartIdx != sliceEndIdx) {//otherwise, adjacent separator or separator at end+ if (sliceStartIdx != sliceEndIdx) { // otherwise, adjacent separator or separator at endbaseIter.setText(text.substring(sliceStartIdx, sliceEndIdx));}}@@ -239,6 +240,4 @@ public class SplittingBreakIterator extends BreakIterator {}return current();}--}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.javaindex 4b93e16af05..53c9e6e068f 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java@@ -17,7 +17,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.PostingsEnum;@@ -28,8 +27,8 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;/*** A filtered LeafReader that only includes the terms that are also in a provided set of terms.- * Certain methods may be unimplemented or cause large operations on the underlying reader- * and be slow.+ * Certain methods may be unimplemented or cause large operations on the underlying reader and be+ * slow.** @lucene.internal*/@@ -40,8 +39,10 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {private final String fieldFilter;/**- * <p>Construct a FilterLeafReader based on the specified base reader.- * <p>Note that base reader is closed if this FilterLeafReader is closed.</p>+ * Construct a FilterLeafReader based on the specified base reader.+ *+ * <p>Note that base reader is closed if this FilterLeafReader is closed.+ ** @param baseLeafReader full/original reader.* @param filterTerms set of terms to filter by -- probably from a TermVector or MemoryIndex.* @param fieldFilter the field to do this on@@ -58,7 +59,7 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {return super.terms(field); // proceed like normal for fields we're not interested in}Terms terms = in.terms(field);- return terms==null ? null : new TermsFilteredTerms(terms, filterTerms);+ return terms == null ? null : new TermsFilteredTerms(terms, filterTerms);}private static final class TermsFilteredTerms extends FilterLeafReader.FilterTerms {@@ -71,9 +72,9 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {this.filterTerms = filterTerms;}- //TODO delegate size() ?+ // TODO delegate size() ?- //TODO delegate getMin, getMax to filterTerms+ // TODO delegate getMin, getMax to filterTerms@Overridepublic TermsEnum iterator() throws IOException {@@ -82,7 +83,8 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {@Overridepublic TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {- return new TermVectorFilteredTermsEnum(in.iterator(), filterTerms.intersect(compiled, startTerm));+ return new TermVectorFilteredTermsEnum(+ in.iterator(), filterTerms.intersect(compiled, startTerm));}}@@ -90,7 +92,8 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {// NOTE: super ("in") is the filteredTermsEnum. This is different than wrappers above because we// navigate the terms using the filter.- //TODO: track the last term state from the term state method and do some potential optimizations+ // TODO: track the last term state from the term state method and do some potential+ // optimizationsprivate final TermsEnum baseTermsEnum;TermVectorFilteredTermsEnum(TermsEnum baseTermsEnum, TermsEnum filteredTermsEnum) {@@ -98,7 +101,7 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {this.baseTermsEnum = baseTermsEnum;}- //TODO delegate docFreq & ttf (moveToCurrentTerm() then call on full?+ // TODO delegate docFreq & ttf (moveToCurrentTerm() then call on full?@Overridepublic PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {@@ -111,10 +114,10 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader {boolean termInBothTermsEnum = baseTermsEnum.seekExact(currentTerm);if (!termInBothTermsEnum) {- throw new IllegalStateException("Term vector term '" + currentTerm.utf8ToString() + "' does not appear in full index.");+ throw new IllegalStateException(+ "Term vector term '" + currentTerm.utf8ToString() + "' does not appear in full index.");}}-}@Overridediff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorOffsetStrategy.javaindex 94528e2c29b..84c0120182d 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorOffsetStrategy.java@@ -17,7 +17,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.Terms;import org.apache.lucene.search.highlight.TermVectorLeafReader;@@ -39,7 +38,8 @@ public class TermVectorOffsetStrategy extends FieldOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {Terms tvTerms = reader.getTermVector(docId, getField());if (tvTerms == null) {return OffsetsEnum.EMPTY;@@ -47,12 +47,6 @@ public class TermVectorOffsetStrategy extends FieldOffsetStrategy {LeafReader singleDocReader = new TermVectorLeafReader(getField(), tvTerms);return createOffsetsEnumFromReader(- new OverlaySingleDocTermsLeafReader(- reader,- singleDocReader,- getField(),- docId),- docId);+ new OverlaySingleDocTermsLeafReader(reader, singleDocReader, getField(), docId), docId);}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.javaindex c8729140296..b99a56915cd 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TokenStreamOffsetStrategy.java@@ -17,7 +17,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;@@ -28,9 +27,10 @@ import org.apache.lucene.util.automaton.Automata;import org.apache.lucene.util.automaton.CharacterRunAutomaton;/**- * Analyzes the text, producing a single {@link OffsetsEnum} wrapping the {@link TokenStream} filtered to terms- * in the query, including wildcards. It can't handle position-sensitive queries (phrases). Passage accuracy suffers- * because the freq() is unknown -- it's always {@link Integer#MAX_VALUE} instead.+ * Analyzes the text, producing a single {@link OffsetsEnum} wrapping the {@link TokenStream}+ * filtered to terms in the query, including wildcards. It can't handle position-sensitive queries+ * (phrases). Passage accuracy suffers because the freq() is unknown -- it's always {@link+ * Integer#MAX_VALUE} instead.*/public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {@@ -42,8 +42,9 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {combinedAutomata = convertTermsToMatchers(components.getTerms(), components.getAutomata());}- //TODO this is inefficient; instead build a union automata just for terms part.- private static CharArrayMatcher[] convertTermsToMatchers(BytesRef[] terms, CharArrayMatcher[] matchers) {+ // TODO this is inefficient; instead build a union automata just for terms part.+ private static CharArrayMatcher[] convertTermsToMatchers(+ BytesRef[] terms, CharArrayMatcher[] matchers) {CharArrayMatcher[] newAutomata = new CharArrayMatcher[terms.length + matchers.length];for (int i = 0; i < terms.length; i++) {String termString = terms[i].utf8ToString();@@ -56,7 +57,8 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {}@Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {return new TokenStreamOffsetsEnum(tokenStream(content), combinedAutomata);}@@ -102,7 +104,6 @@ public class TokenStreamOffsetStrategy extends AnalysisOffsetStrategy {return Integer.MAX_VALUE; // lie}-@Overridepublic int startOffset() throws IOException {return offsetAtt.startOffset();diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UHComponents.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UHComponents.javaindex 65dd84b5fa3..96e40a4cbca 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UHComponents.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UHComponents.java@@ -19,7 +19,6 @@ package org.apache.lucene.search.uhighlight;import java.util.Set;import java.util.function.Predicate;-import org.apache.lucene.search.Query;import org.apache.lucene.util.BytesRef;@@ -32,15 +31,26 @@ public class UHComponents {private final String field;private final Predicate<String> fieldMatcher;private final Query query;- private final BytesRef[] terms; // Query: all terms we extracted (some may be position sensitive)- private final PhraseHelper phraseHelper; // Query: position-sensitive information- private final LabelledCharArrayMatcher[] automata; // Query: wildcards (i.e. multi-term query), not position sensitive- private final boolean hasUnrecognizedQueryPart; // Query: if part of the query (other than the extracted terms / automata) is a leaf we don't know+ // Query: all terms we extracted (some may be position sensitive)+ private final BytesRef[] terms;+ // Query: position-sensitive information+ private final PhraseHelper phraseHelper;+ // Query: wildcards (i.e. multi-term query), not position sensitive+ private final LabelledCharArrayMatcher[] automata;+ // Query: if part of the query (other than the extracted terms / automata) is a leaf we don't know+ private final boolean hasUnrecognizedQueryPart;+private final Set<UnifiedHighlighter.HighlightFlag> highlightFlags;- public UHComponents(String field, Predicate<String> fieldMatcher, Query query,- BytesRef[] terms, PhraseHelper phraseHelper, LabelledCharArrayMatcher[] automata,- boolean hasUnrecognizedQueryPart, Set<UnifiedHighlighter.HighlightFlag> highlightFlags) {+ public UHComponents(+ String field,+ Predicate<String> fieldMatcher,+ Query query,+ BytesRef[] terms,+ PhraseHelper phraseHelper,+ LabelledCharArrayMatcher[] automata,+ boolean hasUnrecognizedQueryPart,+ Set<UnifiedHighlighter.HighlightFlag> highlightFlags) {this.field = field;this.fieldMatcher = fieldMatcher;this.query = query;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.javaindex 45feb03a11f..f84274819e3 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java@@ -33,7 +33,6 @@ import java.util.SortedSet;import java.util.TreeSet;import java.util.function.Predicate;import java.util.function.Supplier;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.document.FieldType;import org.apache.lucene.index.BaseCompositeReader;@@ -65,28 +64,29 @@ import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.InPlaceMergeSorter;/**- * A Highlighter that can get offsets from either- * postings ({@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}),- * term vectors ({@link FieldType#setStoreTermVectorOffsets(boolean)}),- * or via re-analyzing text.- * <p>- * This highlighter treats the single original document as the whole corpus, and then scores individual- * passages as if they were documents in this corpus. It uses a {@link BreakIterator} to find- * passages in the text; by default it breaks using {@link BreakIterator#getSentenceInstance(Locale)- * getSentenceInstance(Locale.ROOT)}. It then iterates in parallel (merge sorting by offset) through- * the positions of all terms from the query, coalescing those hits that occur in a single passage- * into a {@link Passage}, and then scores each Passage using a separate {@link PassageScorer}.- * Passages are finally formatted into highlighted snippets with a {@link PassageFormatter}.- * <p>- * You can customize the behavior by calling some of the setters, or by subclassing and overriding some methods.- * Some important hooks:+ * A Highlighter that can get offsets from either postings ({@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}), term vectors ({@link+ * FieldType#setStoreTermVectorOffsets(boolean)}), or via re-analyzing text.+ *+ * <p>This highlighter treats the single original document as the whole corpus, and then scores+ * individual passages as if they were documents in this corpus. It uses a {@link BreakIterator} to+ * find passages in the text; by default it breaks using {@link+ * BreakIterator#getSentenceInstance(Locale) getSentenceInstance(Locale.ROOT)}. It then iterates in+ * parallel (merge sorting by offset) through the positions of all terms from the query, coalescing+ * those hits that occur in a single passage into a {@link Passage}, and then scores each Passage+ * using a separate {@link PassageScorer}. Passages are finally formatted into highlighted snippets+ * with a {@link PassageFormatter}.+ *+ * <p>You can customize the behavior by calling some of the setters, or by subclassing and+ * overriding some methods. Some important hooks:+ ** <ul>- * <li>{@link #getBreakIterator(String)}: Customize how the text is divided into passages.- * <li>{@link #getScorer(String)}: Customize how passages are ranked.- * <li>{@link #getFormatter(String)}: Customize how snippets are formatted.+ * <li>{@link #getBreakIterator(String)}: Customize how the text is divided into passages.+ * <li>{@link #getScorer(String)}: Customize how passages are ranked.+ * <li>{@link #getFormatter(String)}: Customize how snippets are formatted.* </ul>- * <p>- * This is thread-safe.+ *+ * <p>This is thread-safe.** @lucene.experimental*/@@ -110,7 +110,8 @@ public class UnifiedHighlighter {}}- protected static final LabelledCharArrayMatcher[] ZERO_LEN_AUTOMATA_ARRAY = new LabelledCharArrayMatcher[0];+ protected static final LabelledCharArrayMatcher[] ZERO_LEN_AUTOMATA_ARRAY =+ new LabelledCharArrayMatcher[0];protected final IndexSearcher searcher; // if null, can only use highlightWithoutSearcher@@ -120,12 +121,14 @@ public class UnifiedHighlighter {private boolean defaultHighlightPhrasesStrictly = true; // AKA "accuracy" or "query debugging"- private boolean defaultPassageRelevancyOverSpeed = true; //For analysis, prefer MemoryIndexOffsetStrategy+ // For analysis, prefer MemoryIndexOffsetStrategy+ private boolean defaultPassageRelevancyOverSpeed = true;private int maxLength = DEFAULT_MAX_LENGTH;// BreakIterator is stateful so we use a Supplier factory method- private Supplier<BreakIterator> defaultBreakIterator = () -> BreakIterator.getSentenceInstance(Locale.ROOT);+ private Supplier<BreakIterator> defaultBreakIterator =+ () -> BreakIterator.getSentenceInstance(Locale.ROOT);private Predicate<String> defaultFieldMatcher;@@ -140,9 +143,7 @@ public class UnifiedHighlighter {private int cacheFieldValCharsThreshold = DEFAULT_CACHE_CHARS_THRESHOLD;- /**- * Extracts matching terms after rewriting against an empty index- */+ /** Extracts matching terms after rewriting against an empty index */protected static Set<Term> extractTerms(Query query) throws IOException {Set<Term> queryTerms = new HashSet<>();EMPTY_INDEXSEARCHER.rewrite(query).visit(QueryVisitor.termCollector(queryTerms));@@ -152,15 +153,16 @@ public class UnifiedHighlighter {/*** Constructs the highlighter with the given index searcher and analyzer.*- * @param indexSearcher Usually required, unless {@link #highlightWithoutSearcher(String, Query, String, int)} is- * used, in which case this needs to be null.+ * @param indexSearcher Usually required, unless {@link #highlightWithoutSearcher(String, Query,+ * String, int)} is used, in which case this needs to be null.* @param indexAnalyzer Required, even if in some circumstances it isn't used.*/public UnifiedHighlighter(IndexSearcher indexSearcher, Analyzer indexAnalyzer) {- this.searcher = indexSearcher; //TODO: make non nullable- this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer,- "indexAnalyzer is required"- + " (even if in some circumstances it isn't used)");+ this.searcher = indexSearcher; // TODO: make non nullable+ this.indexAnalyzer =+ Objects.requireNonNull(+ indexAnalyzer,+ "indexAnalyzer is required" + " (even if in some circumstances it isn't used)");}public void setHandleMultiTermQuery(boolean handleMtq) {@@ -205,31 +207,29 @@ public class UnifiedHighlighter {}/**- * Returns whether {@link MultiTermQuery} derivatives will be highlighted. By default it's enabled. MTQ- * highlighting can be expensive, particularly when using offsets in postings.+ * Returns whether {@link MultiTermQuery} derivatives will be highlighted. By default it's+ * enabled. MTQ highlighting can be expensive, particularly when using offsets in postings.*/protected boolean shouldHandleMultiTermQuery(String field) {return defaultHandleMtq;}/**- * Returns whether position sensitive queries (e.g. phrases and {@link SpanQuery}ies)- * should be highlighted strictly based on query matches (slower)- * versus any/all occurrences of the underlying terms. By default it's enabled, but there's no overhead if such- * queries aren't used.+ * Returns whether position sensitive queries (e.g. phrases and {@link SpanQuery}ies) should be+ * highlighted strictly based on query matches (slower) versus any/all occurrences of the+ * underlying terms. By default it's enabled, but there's no overhead if such queries aren't used.*/protected boolean shouldHighlightPhrasesStrictly(String field) {return defaultHighlightPhrasesStrictly;}-protected boolean shouldPreferPassageRelevancyOverSpeed(String field) {return defaultPassageRelevancyOverSpeed;}/**- * Returns the predicate to use for extracting the query part that must be highlighted.- * By default only queries that target the current field are kept. (AKA requireFieldMatch)+ * Returns the predicate to use for extracting the query part that must be highlighted. By default+ * only queries that target the current field are kept. (AKA requireFieldMatch)*/protected Predicate<String> getFieldMatcher(String field) {if (defaultFieldMatcher != null) {@@ -241,42 +241,37 @@ public class UnifiedHighlighter {}/**- * The maximum content size to process. Content will be truncated to this size before highlighting. Typically- * snippets closer to the beginning of the document better summarize its content.+ * The maximum content size to process. Content will be truncated to this size before+ * highlighting. Typically snippets closer to the beginning of the document better summarize its+ * content.*/public int getMaxLength() {return maxLength;}/**- * Returns the {@link BreakIterator} to use for- * dividing text into passages. This returns- * {@link BreakIterator#getSentenceInstance(Locale)} by default;- * subclasses can override to customize.- * <p>- * Note: this highlighter will call- * {@link BreakIterator#preceding(int)} and {@link BreakIterator#next()} many times on it.- * The default generic JDK implementation of {@code preceding} performs poorly.+ * Returns the {@link BreakIterator} to use for dividing text into passages. This returns {@link+ * BreakIterator#getSentenceInstance(Locale)} by default; subclasses can override to customize.+ *+ * <p>Note: this highlighter will call {@link BreakIterator#preceding(int)} and {@link+ * BreakIterator#next()} many times on it. The default generic JDK implementation of {@code+ * preceding} performs poorly.*/protected BreakIterator getBreakIterator(String field) {return defaultBreakIterator.get();}/**- * Returns the {@link PassageScorer} to use for- * ranking passages. This- * returns a new {@code PassageScorer} by default;- * subclasses can override to customize.+ * Returns the {@link PassageScorer} to use for ranking passages. This returns a new {@code+ * PassageScorer} by default; subclasses can override to customize.*/protected PassageScorer getScorer(String field) {return defaultScorer;}/**- * Returns the {@link PassageFormatter} to use for- * formatting passages into highlighted snippets. This- * returns a new {@code PassageFormatter} by default;- * subclasses can override to customize.+ * Returns the {@link PassageFormatter} to use for formatting passages into highlighted snippets.+ * This returns a new {@code PassageFormatter} by default; subclasses can override to customize.*/protected PassageFormatter getFormatter(String field) {return defaultFormatter;@@ -284,70 +279,73 @@ public class UnifiedHighlighter {/*** Returns the number of leading passages (as delineated by the {@link BreakIterator}) when no- * highlights could be found. If it's less than 0 (the default) then this defaults to the {@code maxPassages}- * parameter given for each request. If this is 0 then the resulting highlight is null (not formatted).+ * highlights could be found. If it's less than 0 (the default) then this defaults to the {@code+ * maxPassages} parameter given for each request. If this is 0 then the resulting highlight is+ * null (not formatted).*/protected int getMaxNoHighlightPassages(String field) {return defaultMaxNoHighlightPassages;}/**- * Limits the amount of field value pre-fetching until this threshold is passed. The highlighter- * internally highlights in batches of documents sized on the sum field value length (in chars) of the fields- * to be highlighted (bounded by {@link #getMaxLength()} for each field). By setting this to 0, you can force- * documents to be fetched and highlighted one at a time, which you usually shouldn't do.- * The default is 524288 chars which translates to about a megabyte. However, note- * that the highlighter sometimes ignores this and highlights one document at a time (without caching a- * bunch of documents in advance) when it can detect there's no point in it -- such as when all fields will be- * highlighted via re-analysis as one example.+ * Limits the amount of field value pre-fetching until this threshold is passed. The highlighter+ * internally highlights in batches of documents sized on the sum field value length (in chars) of+ * the fields to be highlighted (bounded by {@link #getMaxLength()} for each field). By setting+ * this to 0, you can force documents to be fetched and highlighted one at a time, which you+ * usually shouldn't do. The default is 524288 chars which translates to about a megabyte.+ * However, note that the highlighter sometimes ignores this and highlights one document at a time+ * (without caching a bunch of documents in advance) when it can detect there's no point in it --+ * such as when all fields will be highlighted via re-analysis as one example.*/public int getCacheFieldValCharsThreshold() { // question: should we size by bytes instead?return cacheFieldValCharsThreshold;}- /**- * ... as passed in from constructor.- */+ /** ... as passed in from constructor. */public IndexSearcher getIndexSearcher() {return searcher;}- /**- * ... as passed in from constructor.- */+ /** ... as passed in from constructor. */public Analyzer getIndexAnalyzer() {return indexAnalyzer;}- /**- * Source of term offsets; essential for highlighting.- */+ /** Source of term offsets; essential for highlighting. */public enum OffsetSource {- POSTINGS, TERM_VECTORS, ANALYSIS, POSTINGS_WITH_TERM_VECTORS, NONE_NEEDED+ POSTINGS,+ TERM_VECTORS,+ ANALYSIS,+ POSTINGS_WITH_TERM_VECTORS,+ NONE_NEEDED}/**- * Determine the offset source for the specified field. The default algorithm is as follows:+ * Determine the offset source for the specified field. The default algorithm is as follows:+ ** <ol>- * <li>This calls {@link #getFieldInfo(String)}. Note this returns null if there is no searcher or if the- * field isn't found there.</li>- * <li> If there's a field info it has- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS} then {@link OffsetSource#POSTINGS} is- * returned.</li>- * <li>If there's a field info and {@link FieldInfo#hasVectors()} then {@link OffsetSource#TERM_VECTORS} is- * returned (note we can't check here if the TV has offsets; if there isn't then an exception will get thrown- * down the line).</li>- * <li>Fall-back: {@link OffsetSource#ANALYSIS} is returned.</li>+ * <li>This calls {@link #getFieldInfo(String)}. Note this returns null if there is no searcher+ * or if the field isn't found there.+ * <li>If there's a field info it has {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS} then {@link OffsetSource#POSTINGS}+ * is returned.+ * <li>If there's a field info and {@link FieldInfo#hasVectors()} then {@link+ * OffsetSource#TERM_VECTORS} is returned (note we can't check here if the TV has offsets;+ * if there isn't then an exception will get thrown down the line).+ * <li>Fall-back: {@link OffsetSource#ANALYSIS} is returned.* </ol>- * <p>- * Note that the highlighter sometimes switches to something else based on the query, such as if you have- * {@link OffsetSource#POSTINGS_WITH_TERM_VECTORS} but in fact don't need term vectors.+ *+ * <p>Note that the highlighter sometimes switches to something else based on the query, such as+ * if you have {@link OffsetSource#POSTINGS_WITH_TERM_VECTORS} but in fact don't need term+ * vectors.*/protected OffsetSource getOffsetSource(String field) {FieldInfo fieldInfo = getFieldInfo(field);if (fieldInfo != null) {if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {- return fieldInfo.hasVectors() ? OffsetSource.POSTINGS_WITH_TERM_VECTORS : OffsetSource.POSTINGS;+ return fieldInfo.hasVectors()+ ? OffsetSource.POSTINGS_WITH_TERM_VECTORS+ : OffsetSource.POSTINGS;}if (fieldInfo.hasVectors()) { // unfortunately we can't also check if the TV has offsetsreturn OffsetSource.TERM_VECTORS;@@ -357,14 +355,15 @@ public class UnifiedHighlighter {}/**- * Called by the default implementation of {@link #getOffsetSource(String)}.- * If there is no searcher then we simply always return null.+ * Called by the default implementation of {@link #getOffsetSource(String)}. If there is no+ * searcher then we simply always return null.*/protected FieldInfo getFieldInfo(String field) {if (searcher == null) {return null;}- // Need thread-safety for lazy-init but lets avoid 'synchronized' by using double-check locking idiom+ // Need thread-safety for lazy-init but lets avoid 'synchronized' by using double-check locking+ // idiomFieldInfos fieldInfos = this.fieldInfos; // note: it's volatile; read onceif (fieldInfos == null) {synchronized (this) {@@ -373,9 +372,7 @@ public class UnifiedHighlighter {fieldInfos = FieldInfos.getMergedFieldInfos(searcher.getIndexReader());this.fieldInfos = fieldInfos;}-}-}return fieldInfos.fieldInfo(field);}@@ -383,16 +380,15 @@ public class UnifiedHighlighter {/*** Highlights the top passages from a single field.*- * @param field field name to highlight.- * Must have a stored string value and also be indexed with offsets.- * @param query query to highlight.+ * @param field field name to highlight. Must have a stored string value and also be indexed with+ * offsets.+ * @param query query to highlight.* @param topDocs TopDocs containing the summary result documents to highlight.- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.- * If no highlights were found for a document, the- * first sentence for the field will be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>. If+ * no highlights were found for a document, the first sentence for the field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/public String[] highlight(String field, Query query, TopDocs topDocs) throws IOException {return highlight(field, query, topDocs, 1);@@ -401,28 +397,30 @@ public class UnifiedHighlighter {/*** Highlights the top-N passages from a single field.*- * @param field field name to highlight. Must have a stored string value.- * @param query query to highlight.- * @param topDocs TopDocs containing the summary result documents to highlight.- * @param maxPassages The maximum number of top-N ranked passages used to- * form the highlighted snippets.- * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>.- * If no highlights were found for a document, the- * first {@code maxPassages} sentences from the- * field will be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @param field field name to highlight. Must have a stored string value.+ * @param query query to highlight.+ * @param topDocs TopDocs containing the summary result documents to highlight.+ * @param maxPassages The maximum number of top-N ranked passages used to form the highlighted+ * snippets.+ * @return Array of formatted snippets corresponding to the documents in <code>topDocs</code>. If+ * no highlights were found for a document, the first {@code maxPassages} sentences from the+ * field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/- public String[] highlight(String field, Query query, TopDocs topDocs, int maxPassages) throws IOException {- Map<String, String[]> res = highlightFields(new String[]{field}, query, topDocs, new int[]{maxPassages});+ public String[] highlight(String field, Query query, TopDocs topDocs, int maxPassages)+ throws IOException {+ Map<String, String[]> res =+ highlightFields(new String[] {field}, query, topDocs, new int[] {maxPassages});return res.get(field);}/*** Highlights the top passages from multiple fields.- * <p>- * Conceptually, this behaves as a more efficient form of:+ *+ * <p>Conceptually, this behaves as a more efficient form of:+ ** <pre class="prettyprint">* Map m = new HashMap();* for (String field : fields) {@@ -431,18 +429,18 @@ public class UnifiedHighlighter {* return m;* </pre>*- * @param fields field names to highlight. Must have a stored string value.- * @param query query to highlight.+ * @param fields field names to highlight. Must have a stored string value.+ * @param query query to highlight.* @param topDocs TopDocs containing the summary result documents to highlight.- * @return Map keyed on field name, containing the array of formatted snippets- * corresponding to the documents in <code>topDocs</code>.- * If no highlights were found for a document, the- * first sentence from the field will be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @return Map keyed on field name, containing the array of formatted snippets corresponding to+ * the documents in <code>topDocs</code>. If no highlights were found for a document, the+ * first sentence from the field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/- public Map<String, String[]> highlightFields(String[] fields, Query query, TopDocs topDocs) throws IOException {+ public Map<String, String[]> highlightFields(String[] fields, Query query, TopDocs topDocs)+ throws IOException {int maxPassages[] = new int[fields.length];Arrays.fill(maxPassages, 1);return highlightFields(fields, query, topDocs, maxPassages);@@ -450,8 +448,9 @@ public class UnifiedHighlighter {/*** Highlights the top-N passages from multiple fields.- * <p>- * Conceptually, this behaves as a more efficient form of:+ *+ * <p>Conceptually, this behaves as a more efficient form of:+ ** <pre class="prettyprint">* Map m = new HashMap();* for (String field : fields) {@@ -460,22 +459,20 @@ public class UnifiedHighlighter {* return m;* </pre>*- * @param fields field names to highlight. Must have a stored string value.- * @param query query to highlight.- * @param topDocs TopDocs containing the summary result documents to highlight.- * @param maxPassages The maximum number of top-N ranked passages per-field used to- * form the highlighted snippets.- * @return Map keyed on field name, containing the array of formatted snippets- * corresponding to the documents in <code>topDocs</code>.- * If no highlights were found for a document, the- * first {@code maxPassages} sentences from the- * field will be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @param fields field names to highlight. Must have a stored string value.+ * @param query query to highlight.+ * @param topDocs TopDocs containing the summary result documents to highlight.+ * @param maxPassages The maximum number of top-N ranked passages per-field used to form the+ * highlighted snippets.+ * @return Map keyed on field name, containing the array of formatted snippets corresponding to+ * the documents in <code>topDocs</code>. If no highlights were found for a document, the+ * first {@code maxPassages} sentences from the field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/- public Map<String, String[]> highlightFields(String[] fields, Query query, TopDocs topDocs, int[] maxPassages)- throws IOException {+ public Map<String, String[]> highlightFields(+ String[] fields, Query query, TopDocs topDocs, int[] maxPassages) throws IOException {final ScoreDoc scoreDocs[] = topDocs.scoreDocs;int docids[] = new int[scoreDocs.length];for (int i = 0; i < docids.length; i++) {@@ -486,27 +483,25 @@ public class UnifiedHighlighter {}/**- * Highlights the top-N passages from multiple fields,- * for the provided int[] docids.+ * Highlights the top-N passages from multiple fields, for the provided int[] docids.*- * @param fieldsIn field names to highlight. Must have a stored string value.- * @param query query to highlight.- * @param docidsIn containing the document IDs to highlight.- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to- * form the highlighted snippets.- * @return Map keyed on field name, containing the array of formatted snippets- * corresponding to the documents in <code>docidsIn</code>.- * If no highlights were found for a document, the- * first {@code maxPassages} from the field will- * be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @param fieldsIn field names to highlight. Must have a stored string value.+ * @param query query to highlight.+ * @param docidsIn containing the document IDs to highlight.+ * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to form the+ * highlighted snippets.+ * @return Map keyed on field name, containing the array of formatted snippets corresponding to+ * the documents in <code>docidsIn</code>. If no highlights were found for a document, the+ * first {@code maxPassages} from the field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/- public Map<String, String[]> highlightFields(String[] fieldsIn, Query query, int[] docidsIn, int[] maxPassagesIn)- throws IOException {+ public Map<String, String[]> highlightFields(+ String[] fieldsIn, Query query, int[] docidsIn, int[] maxPassagesIn) throws IOException {Map<String, String[]> snippets = new HashMap<>();- for (Map.Entry<String, Object[]> ent : highlightFieldsAsObjects(fieldsIn, query, docidsIn, maxPassagesIn).entrySet()) {+ for (Map.Entry<String, Object[]> ent :+ highlightFieldsAsObjects(fieldsIn, query, docidsIn, maxPassagesIn).entrySet()) {Object[] snippetObjects = ent.getValue();String[] snippetStrings = new String[snippetObjects.length];snippets.put(ent.getKey(), snippetStrings);@@ -522,27 +517,24 @@ public class UnifiedHighlighter {}/**- * Expert: highlights the top-N passages from multiple fields,- * for the provided int[] docids, to custom Object as- * returned by the {@link PassageFormatter}. Use- * this API to render to something other than String.+ * Expert: highlights the top-N passages from multiple fields, for the provided int[] docids, to+ * custom Object as returned by the {@link PassageFormatter}. Use this API to render to something+ * other than String.*- * @param fieldsIn field names to highlight. Must have a stored string value.- * @param query query to highlight.- * @param docIdsIn containing the document IDs to highlight.- * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to- * form the highlighted snippets.- * @return Map keyed on field name, containing the array of formatted snippets- * corresponding to the documents in <code>docIdsIn</code>.- * If no highlights were found for a document, the- * first {@code maxPassages} from the field will- * be returned.- * @throws IOException if an I/O error occurred during processing- * @throws IllegalArgumentException if <code>field</code> was indexed without- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}+ * @param fieldsIn field names to highlight. Must have a stored string value.+ * @param query query to highlight.+ * @param docIdsIn containing the document IDs to highlight.+ * @param maxPassagesIn The maximum number of top-N ranked passages per-field used to form the+ * highlighted snippets.+ * @return Map keyed on field name, containing the array of formatted snippets corresponding to+ * the documents in <code>docIdsIn</code>. If no highlights were found for a document, the+ * first {@code maxPassages} from the field will be returned.+ * @throws IOException if an I/O error occurred during processing+ * @throws IllegalArgumentException if <code>field</code> was indexed without {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}*/- protected Map<String, Object[]> highlightFieldsAsObjects(String[] fieldsIn, Query query, int[] docIdsIn,- int[] maxPassagesIn) throws IOException {+ protected Map<String, Object[]> highlightFieldsAsObjects(+ String[] fieldsIn, Query query, int[] docIdsIn, int[] maxPassagesIn) throws IOException {if (fieldsIn.length < 1) {throw new IllegalArgumentException("fieldsIn must not be empty");}@@ -550,8 +542,9 @@ public class UnifiedHighlighter {throw new IllegalArgumentException("invalid number of maxPassagesIn");}if (searcher == null) {- throw new IllegalStateException("This method requires that an indexSearcher was passed in the "- + "constructor. Perhaps you mean to call highlightWithoutSearcher?");+ throw new IllegalStateException(+ "This method requires that an indexSearcher was passed in the "+ + "constructor. Perhaps you mean to call highlightWithoutSearcher?");}// Sort docs & fields for sequential i/o@@ -564,7 +557,8 @@ public class UnifiedHighlighter {// Sort fields w/ maxPassages pair: (copy input arrays since we sort in-place)final String fields[] = new String[fieldsIn.length];final int maxPassages[] = new int[maxPassagesIn.length];- copyAndSortFieldsWithMaxPassages(fieldsIn, maxPassagesIn, fields, maxPassages); // latter 2 are "out" params+ copyAndSortFieldsWithMaxPassages(+ fieldsIn, maxPassagesIn, fields, maxPassages); // latter 2 are "out" params// Init field highlighters (where most of the highlight logic lives, and on a per field basis)Set<Term> queryTerms = extractTerms(query);@@ -572,7 +566,8 @@ public class UnifiedHighlighter {int numTermVectors = 0;int numPostings = 0;for (int f = 0; f < fields.length; f++) {- FieldHighlighter fieldHighlighter = getFieldHighlighter(fields[f], query, queryTerms, maxPassages[f]);+ FieldHighlighter fieldHighlighter =+ getFieldHighlighter(fields[f], query, queryTerms, maxPassages[f]);fieldHighlighters[f] = fieldHighlighter;switch (fieldHighlighter.getOffsetSource()) {@@ -589,7 +584,7 @@ public class UnifiedHighlighter {case ANALYSIS:case NONE_NEEDED:default:- //do nothing+ // do nothingbreak;}}@@ -604,24 +599,24 @@ public class UnifiedHighlighter {// Highlight in doc batches determined by loadFieldValues (consumes from docIdIter)DocIdSetIterator docIdIter = asDocIdSetIterator(docIds);for (int batchDocIdx = 0; batchDocIdx < docIds.length; ) {- // Load the field values of the first batch of document(s) (note: commonly all docs are in this batch)- List<CharSequence[]> fieldValsByDoc =- loadFieldValues(fields, docIdIter, cacheCharsThreshold);+ // Load the field values of the first batch of document(s) (note: commonly all docs are in+ // this batch)+ List<CharSequence[]> fieldValsByDoc = loadFieldValues(fields, docIdIter, cacheCharsThreshold);// the size of the above list is the size of the batch (num of docs in the batch)// Highlight in per-field order first, then by doc (better I/O pattern)for (int fieldIdx = 0; fieldIdx < fields.length; fieldIdx++) {- Object[] resultByDocIn = highlightDocsInByField[fieldIdx];//parallel to docIdsIn+ Object[] resultByDocIn = highlightDocsInByField[fieldIdx]; // parallel to docIdsInFieldHighlighter fieldHighlighter = fieldHighlighters[fieldIdx];for (int docIdx = batchDocIdx; docIdx - batchDocIdx < fieldValsByDoc.size(); docIdx++) {- int docId = docIds[docIdx];//sorted order+ int docId = docIds[docIdx]; // sorted orderCharSequence content = fieldValsByDoc.get(docIdx - batchDocIdx)[fieldIdx];if (content == null) {continue;}IndexReader indexReader =(fieldHighlighter.getOffsetSource() == OffsetSource.TERM_VECTORS- && indexReaderWithTermVecCache != null)+ && indexReaderWithTermVecCache != null)? indexReaderWithTermVecCache: searcher.getIndexReader();final LeafReader leafReader;@@ -633,13 +628,11 @@ public class UnifiedHighlighter {leafReader = leafReaderContext.reader();docId -= leafReaderContext.docBase; // adjust 'doc' to be within this leaf reader}- int docInIndex = docInIndexes[docIdx];//original input order+ int docInIndex = docInIndexes[docIdx]; // original input orderassert resultByDocIn[docInIndex] == null;resultByDocIn[docInIndex] =- fieldHighlighter- .highlightFieldForDoc(leafReader, docId, content.toString());+ fieldHighlighter.highlightFieldForDoc(leafReader, docId, content.toString());}-}batchDocIdx += fieldValsByDoc.size();@@ -648,7 +641,8 @@ public class UnifiedHighlighter {assert docIdIter.docID() == DocIdSetIterator.NO_MORE_DOCS|| docIdIter.nextDoc() == DocIdSetIterator.NO_MORE_DOCS;- // TODO reconsider the return type; since this is an "advanced" method, lets not return a Map? Notice the only+ // TODO reconsider the return type; since this is an "advanced" method, lets not return a Map?+ // Notice the only// caller simply iterates it to build another structure.// field -> object highlights parallel to docIdsIn@@ -660,8 +654,8 @@ public class UnifiedHighlighter {}/**- * When cacheCharsThreshold is 0, loadFieldValues() only fetches one document at a time. We override it to be 0- * in two circumstances:+ * When cacheCharsThreshold is 0, loadFieldValues() only fetches one document at a time. We+ * override it to be 0 in two circumstances:*/private int calculateOptimalCacheCharsThreshold(int numTermVectors, int numPostings) {if (numPostings == 0 && numTermVectors == 0) {@@ -669,17 +663,20 @@ public class UnifiedHighlighter {// because no other info on disk is needed to highlight it.return 0;} else if (numTermVectors >= 2) {- // (2) When two or more fields have term vectors, given the field-then-doc algorithm, the underlying term- // vectors will be fetched in a terrible access pattern unless we highlight a doc at a time and use a special- // current-doc TV cache. So we do that. Hopefully one day TVs will be improved to make this pointless.+ // (2) When two or more fields have term vectors, given the field-then-doc algorithm, the+ // underlying term+ // vectors will be fetched in a terrible access pattern unless we highlight a doc at a time+ // and use a special+ // current-doc TV cache. So we do that. Hopefully one day TVs will be improved to make this+ // pointless.return 0;} else {return getCacheFieldValCharsThreshold();}}- private void copyAndSortFieldsWithMaxPassages(String[] fieldsIn, int[] maxPassagesIn, final String[] fields,- final int[] maxPassages) {+ private void copyAndSortFieldsWithMaxPassages(+ String[] fieldsIn, int[] maxPassagesIn, final String[] fields, final int[] maxPassages) {System.arraycopy(fieldsIn, 0, fields, 0, fieldsIn.length);System.arraycopy(maxPassagesIn, 0, maxPassages, 0, maxPassagesIn.length);new InPlaceMergeSorter() {@@ -697,11 +694,11 @@ public class UnifiedHighlighter {protected int compare(int i, int j) {return fields[i].compareTo(fields[j]);}-}.sort(0, fields.length);}- private void copyAndSortDocIdsWithIndex(int[] docIdsIn, final int[] docIds, final int[] docInIndexes) {+ private void copyAndSortDocIdsWithIndex(+ int[] docIdsIn, final int[] docIds, final int[] docInIndexes) {System.arraycopy(docIdsIn, 0, docIds, 0, docIdsIn.length);for (int i = 0; i < docInIndexes.length; i++) {docInIndexes[i] = i;@@ -725,26 +722,30 @@ public class UnifiedHighlighter {}/**- * Highlights text passed as a parameter. This requires the {@link IndexSearcher} provided to this highlighter is- * null. This use-case is more rare. Naturally, the mode of operation will be {@link OffsetSource#ANALYSIS}.- * The result of this method is whatever the {@link PassageFormatter} returns. For the {@link- * DefaultPassageFormatter} and assuming {@code content} has non-zero length, the result will be a non-null- * string -- so it's safe to call {@link Object#toString()} on it in that case.+ * Highlights text passed as a parameter. This requires the {@link IndexSearcher} provided to this+ * highlighter is null. This use-case is more rare. Naturally, the mode of operation will be+ * {@link OffsetSource#ANALYSIS}. The result of this method is whatever the {@link+ * PassageFormatter} returns. For the {@link DefaultPassageFormatter} and assuming {@code content}+ * has non-zero length, the result will be a non-null string -- so it's safe to call {@link+ * Object#toString()} on it in that case.*- * @param field field name to highlight (as found in the query).- * @param query query to highlight.- * @param content text to highlight.- * @param maxPassages The maximum number of top-N ranked passages used to- * form the highlighted snippets.- * @return result of the {@link PassageFormatter} -- probably a String. Might be null.+ * @param field field name to highlight (as found in the query).+ * @param query query to highlight.+ * @param content text to highlight.+ * @param maxPassages The maximum number of top-N ranked passages used to form the highlighted+ * snippets.+ * @return result of the {@link PassageFormatter} -- probably a String. Might be null.* @throws IOException if an I/O error occurred during processing*/- //TODO make content a List? and return a List? and ensure getEmptyHighlight is never invoked multiple times?+ // TODO make content a List? and return a List? and ensure getEmptyHighlight is never invoked+ // multiple times?public Object highlightWithoutSearcher(String field, Query query, String content, int maxPassages)throws IOException {if (this.searcher != null) {- throw new IllegalStateException("highlightWithoutSearcher should only be called on a " +- getClass().getSimpleName() + " without an IndexSearcher.");+ throw new IllegalStateException(+ "highlightWithoutSearcher should only be called on a "+ + getClass().getSimpleName()+ + " without an IndexSearcher.");}Objects.requireNonNull(content, "content is required");Set<Term> queryTerms = extractTerms(query);@@ -752,10 +753,12 @@ public class UnifiedHighlighter {.highlightFieldForDoc(null, -1, content);}- protected FieldHighlighter getFieldHighlighter(String field, Query query, Set<Term> allTerms, int maxPassages) {+ protected FieldHighlighter getFieldHighlighter(+ String field, Query query, Set<Term> allTerms, int maxPassages) {UHComponents components = getHighlightComponents(field, query, allTerms);OffsetSource offsetSource = getOptimizedOffsetSource(components);- return new FieldHighlighter(field,+ return new FieldHighlighter(+ field,getOffsetStrategy(offsetSource, components),new SplittingBreakIterator(getBreakIterator(field), UnifiedHighlighter.MULTIVAL_SEP_CHAR),getScorer(field),@@ -775,31 +778,41 @@ public class UnifiedHighlighter {terms = filterExtractedTerms(fieldMatcher, allTerms);automata = getAutomata(field, query, highlightFlags);} // otherwise don't need to extract- return new UHComponents(field, fieldMatcher, query, terms, phraseHelper, automata, queryHasUnrecognizedPart, highlightFlags);+ return new UHComponents(+ field,+ fieldMatcher,+ query,+ terms,+ phraseHelper,+ automata,+ queryHasUnrecognizedPart,+ highlightFlags);}protected boolean hasUnrecognizedQuery(Predicate<String> fieldMatcher, Query query) {boolean[] hasUnknownLeaf = new boolean[1];- query.visit(new QueryVisitor() {- @Override- public boolean acceptField(String field) {- // checking hasUnknownLeaf is a trick to exit early- return hasUnknownLeaf[0] == false && fieldMatcher.test(field);- }+ query.visit(+ new QueryVisitor() {+ @Override+ public boolean acceptField(String field) {+ // checking hasUnknownLeaf is a trick to exit early+ return hasUnknownLeaf[0] == false && fieldMatcher.test(field);+ }- @Override- public void visitLeaf(Query query) {- if (MultiTermHighlighting.canExtractAutomataFromLeafQuery(query) == false) {- if (!(query instanceof MatchAllDocsQuery || query instanceof MatchNoDocsQuery)) {- hasUnknownLeaf[0] = true;+ @Override+ public void visitLeaf(Query query) {+ if (MultiTermHighlighting.canExtractAutomataFromLeafQuery(query) == false) {+ if (!(query instanceof MatchAllDocsQuery || query instanceof MatchNoDocsQuery)) {+ hasUnknownLeaf[0] = true;+ }+ }}- }- }- });+ });return hasUnknownLeaf[0];}- protected static BytesRef[] filterExtractedTerms(Predicate<String> fieldMatcher, Set<Term> queryTerms) {+ protected static BytesRef[] filterExtractedTerms(+ Predicate<String> fieldMatcher, Set<Term> queryTerms) {// Strip off the redundant field and sort the remaining termsSortedSet<BytesRef> filteredTerms = new TreeSet<>();for (Term term : queryTerms) {@@ -824,28 +837,34 @@ public class UnifiedHighlighter {return highlightFlags;}- protected PhraseHelper getPhraseHelper(String field, Query query, Set<HighlightFlag> highlightFlags) {+ protected PhraseHelper getPhraseHelper(+ String field, Query query, Set<HighlightFlag> highlightFlags) {boolean useWeightMatchesIter = highlightFlags.contains(HighlightFlag.WEIGHT_MATCHES);if (useWeightMatchesIter) {return PhraseHelper.NONE; // will be handled by Weight.matches which always considers phrases}boolean highlightPhrasesStrictly = highlightFlags.contains(HighlightFlag.PHRASES);boolean handleMultiTermQuery = highlightFlags.contains(HighlightFlag.MULTI_TERM_QUERY);- return highlightPhrasesStrictly ?- new PhraseHelper(query, field, getFieldMatcher(field),+ return highlightPhrasesStrictly+ ? new PhraseHelper(+ query,+ field,+ getFieldMatcher(field),this::requiresRewrite,this::preSpanQueryRewrite,- !handleMultiTermQuery- )+ !handleMultiTermQuery): PhraseHelper.NONE;}- protected LabelledCharArrayMatcher[] getAutomata(String field, Query query, Set<HighlightFlag> highlightFlags) {- // do we "eagerly" look in span queries for automata here, or do we not and let PhraseHelper handle those?+ protected LabelledCharArrayMatcher[] getAutomata(+ String field, Query query, Set<HighlightFlag> highlightFlags) {+ // do we "eagerly" look in span queries for automata here, or do we not and let PhraseHelper+ // handle those?// if don't highlight phrases strictly,final boolean lookInSpan =!highlightFlags.contains(HighlightFlag.PHRASES) // no PhraseHelper- || highlightFlags.contains(HighlightFlag.WEIGHT_MATCHES); // Weight.Matches will find all+ || highlightFlags.contains(+ HighlightFlag.WEIGHT_MATCHES); // Weight.Matches will find allreturn highlightFlags.contains(HighlightFlag.MULTI_TERM_QUERY)? MultiTermHighlighting.extractAutomata(query, getFieldMatcher(field), lookInSpan)@@ -856,43 +875,50 @@ public class UnifiedHighlighter {OffsetSource offsetSource = getOffsetSource(components.getField());// null automata means unknown, so assume a possibility- boolean mtqOrRewrite = components.getAutomata() == null || components.getAutomata().length > 0- || components.getPhraseHelper().willRewrite() || components.hasUnrecognizedQueryPart();+ boolean mtqOrRewrite =+ components.getAutomata() == null+ || components.getAutomata().length > 0+ || components.getPhraseHelper().willRewrite()+ || components.hasUnrecognizedQueryPart();// null terms means unknown, so assume something to highlight- if (mtqOrRewrite == false && components.getTerms() != null && components.getTerms().length == 0) {- return OffsetSource.NONE_NEEDED; //nothing to highlight+ if (mtqOrRewrite == false+ && components.getTerms() != null+ && components.getTerms().length == 0) {+ return OffsetSource.NONE_NEEDED; // nothing to highlight}switch (offsetSource) {case POSTINGS:- if (mtqOrRewrite) { // may need to see scan through all terms for the highlighted document efficiently+ if (mtqOrRewrite) { // may need to see scan through all terms for the highlighted document+ // efficientlyreturn OffsetSource.ANALYSIS;}break;case POSTINGS_WITH_TERM_VECTORS:if (mtqOrRewrite == false) {- return OffsetSource.POSTINGS; //We don't need term vectors+ return OffsetSource.POSTINGS; // We don't need term vectors}break;case ANALYSIS:case TERM_VECTORS:case NONE_NEEDED:default:- //stick with the original offset source+ // stick with the original offset sourcebreak;}return offsetSource;}- protected FieldOffsetStrategy getOffsetStrategy(OffsetSource offsetSource, UHComponents components) {+ protected FieldOffsetStrategy getOffsetStrategy(+ OffsetSource offsetSource, UHComponents components) {switch (offsetSource) {case ANALYSIS:- if (!components.getPhraseHelper().hasPositionSensitivity() &&- !components.getHighlightFlags().contains(HighlightFlag.PASSAGE_RELEVANCY_OVER_SPEED) &&- !components.getHighlightFlags().contains(HighlightFlag.WEIGHT_MATCHES)) {- //skip using a memory index since it's pure term filtering+ if (!components.getPhraseHelper().hasPositionSensitivity()+ && !components.getHighlightFlags().contains(HighlightFlag.PASSAGE_RELEVANCY_OVER_SPEED)+ && !components.getHighlightFlags().contains(HighlightFlag.WEIGHT_MATCHES)) {+ // skip using a memory index since it's pure term filteringreturn new TokenStreamOffsetStrategy(components, getIndexAnalyzer());} else {return new MemoryIndexOffsetStrategy(components, getIndexAnalyzer());@@ -912,20 +938,22 @@ public class UnifiedHighlighter {/*** When highlighting phrases accurately, we need to know which {@link SpanQuery}'s need to have- * {@link Query#rewrite(IndexReader)} called on them. It helps performance to avoid it if it's not needed.- * This method will be invoked on all SpanQuery instances recursively. If you have custom SpanQuery queries then- * override this to check instanceof and provide a definitive answer. If the query isn't your custom one, simply- * return null to have the default rules apply, which govern the ones included in Lucene.+ * {@link Query#rewrite(IndexReader)} called on them. It helps performance to avoid it if it's not+ * needed. This method will be invoked on all SpanQuery instances recursively. If you have custom+ * SpanQuery queries then override this to check instanceof and provide a definitive answer. If+ * the query isn't your custom one, simply return null to have the default rules apply, which+ * govern the ones included in Lucene.*/protected Boolean requiresRewrite(SpanQuery spanQuery) {return null;}/**- * When highlighting phrases accurately, we may need to handle custom queries that aren't supported in the- * {@link org.apache.lucene.search.highlight.WeightedSpanTermExtractor} as called by the {@code PhraseHelper}.- * Should custom query types be needed, this method should be overriden to return a collection of queries if appropriate,- * or null if nothing to do. If the query is not custom, simply returning null will allow the default rules to apply.+ * When highlighting phrases accurately, we may need to handle custom queries that aren't+ * supported in the {@link org.apache.lucene.search.highlight.WeightedSpanTermExtractor} as called+ * by the {@code PhraseHelper}. Should custom query types be needed, this method should be+ * overriden to return a collection of queries if appropriate, or null if nothing to do. If the+ * query is not custom, simply returning null will allow the default rules to apply.** @param query Query to be highlighted* @return A Collection of Query object(s) if needs to be rewritten, otherwise null.@@ -965,17 +993,17 @@ public class UnifiedHighlighter {}/**- * Loads the String values for each docId by field to be highlighted. By default this loads from stored fields- * by the same name as given, but a subclass can change the source. The returned Strings must be identical to- * what was indexed (at least for postings or term-vectors offset sources).- * This method must load fields for at least one document from the given {@link DocIdSetIterator}- * but need not return all of them; by default the character lengths are summed and this method will return early- * when {@code cacheCharsThreshold} is exceeded. Specifically if that number is 0, then only one document is- * fetched no matter what. Values in the array of {@link CharSequence} will be null if no value was found.+ * Loads the String values for each docId by field to be highlighted. By default this loads from+ * stored fields by the same name as given, but a subclass can change the source. The returned+ * Strings must be identical to what was indexed (at least for postings or term-vectors offset+ * sources). This method must load fields for at least one document from the given {@link+ * DocIdSetIterator} but need not return all of them; by default the character lengths are summed+ * and this method will return early when {@code cacheCharsThreshold} is exceeded. Specifically if+ * that number is 0, then only one document is fetched no matter what. Values in the array of+ * {@link CharSequence} will be null if no value was found.*/- protected List<CharSequence[]> loadFieldValues(String[] fields,- DocIdSetIterator docIter, int cacheCharsThreshold)- throws IOException {+ protected List<CharSequence[]> loadFieldValues(+ String[] fields, DocIdSetIterator docIter, int cacheCharsThreshold) throws IOException {List<CharSequence[]> docListOfFields =new ArrayList<>(cacheCharsThreshold == 0 ? 1 : (int) Math.min(64, docIter.cost()));@@ -997,22 +1025,22 @@ public class UnifiedHighlighter {return docListOfFields;}- /**- * @lucene.internal- */+ /** @lucene.internal */protected LimitedStoredFieldVisitor newLimitedStoredFieldsVisitor(String[] fields) {return new LimitedStoredFieldVisitor(fields, MULTIVAL_SEP_CHAR, getMaxLength());}/**- * Fetches stored fields for highlighting. Uses a multi-val separator char and honors a max length to retrieve.+ * Fetches stored fields for highlighting. Uses a multi-val separator char and honors a max length+ * to retrieve.+ ** @lucene.internal*/protected static class LimitedStoredFieldVisitor extends StoredFieldVisitor {protected final String[] fields;protected final char valueSeparator;protected final int maxLength;- protected CharSequence[] values;// starts off as String; may become StringBuilder.+ protected CharSequence[] values; // starts off as String; may become StringBuilder.protected int currentField;public LimitedStoredFieldVisitor(String[] fields, char valueSeparator, int maxLength) {@@ -1032,9 +1060,11 @@ public class UnifiedHighlighter {Objects.requireNonNull(value, "String value should not be null");CharSequence curValue = values[currentField];if (curValue == null) {- //question: if truncate due to maxLength, should we try and avoid keeping the other chars in-memory on+ // question: if truncate due to maxLength, should we try and avoid keeping the other chars+ // in-memory on// the backing char[]?- values[currentField] = value.substring(0, Math.min(maxLength, value.length()));//note: may return 'this'+ values[currentField] =+ value.substring(0, Math.min(maxLength, value.length())); // note: may return 'this'return;}final int lengthBudget = maxLength - curValue.length();@@ -1046,7 +1076,8 @@ public class UnifiedHighlighter {curValueBuilder = (StringBuilder) curValue;} else {// upgrade String to StringBuilder. Choose a good initial size.- curValueBuilder = new StringBuilder(curValue.length() + Math.min(lengthBudget, value.length() + 256));+ curValueBuilder =+ new StringBuilder(curValue.length() + Math.min(lengthBudget, value.length() + 256));curValueBuilder.append(curValue);}curValueBuilder.append(valueSeparator);@@ -1070,21 +1101,21 @@ public class UnifiedHighlighter {CharSequence[] getValuesByField() {return this.values;}-}/**- * Wraps an IndexReader that remembers/caches the last call to {@link LeafReader#getTermVectors(int)} so that- * if the next call has the same ID, then it is reused. If TV's were column-stride (like doc-values), there would- * be no need for this.+ * Wraps an IndexReader that remembers/caches the last call to {@link+ * LeafReader#getTermVectors(int)} so that if the next call has the same ID, then it is reused. If+ * TV's were column-stride (like doc-values), there would be no need for this.*/private static class TermVectorReusingLeafReader extends FilterLeafReader {static IndexReader wrap(IndexReader reader) throws IOException {- LeafReader[] leafReaders = reader.leaves().stream()- .map(LeafReaderContext::reader)- .map(TermVectorReusingLeafReader::new)- .toArray(LeafReader[]::new);+ LeafReader[] leafReaders =+ reader.leaves().stream()+ .map(LeafReaderContext::reader)+ .map(TermVectorReusingLeafReader::new)+ .toArray(LeafReader[]::new);return new BaseCompositeReader<IndexReader>(leafReaders) {@Overrideprotected void doClose() { // don't close the underlying reader@@ -1122,12 +1153,9 @@ public class UnifiedHighlighter {public CacheHelper getReaderCacheHelper() {return null;}-}- /**- * Flags for controlling highlighting behavior.- */+ /** Flags for controlling highlighting behavior. */public enum HighlightFlag {/** @see UnifiedHighlighter#setHighlightPhrasesStrictly(boolean) */PHRASES,@@ -1135,14 +1163,14 @@ public class UnifiedHighlighter {/** @see UnifiedHighlighter#setHandleMultiTermQuery(boolean) */MULTI_TERM_QUERY,- /** Passage relevancy is more important than speed. True by default. */+ /** Passage relevancy is more important than speed. True by default. */PASSAGE_RELEVANCY_OVER_SPEED,/**- * Internally use the {@link Weight#matches(LeafReaderContext, int)} API for highlighting.- * It's more accurate to the query, though might not calculate passage relevancy as well.- * Use of this flag requires {@link #MULTI_TERM_QUERY} and {@link #PHRASES}.- * {@link #PASSAGE_RELEVANCY_OVER_SPEED} will be ignored. False by default.+ * Internally use the {@link Weight#matches(LeafReaderContext, int)} API for highlighting. It's+ * more accurate to the query, though might not calculate passage relevancy as well. Use of this+ * flag requires {@link #MULTI_TERM_QUERY} and {@link #PHRASES}. {@link+ * #PASSAGE_RELEVANCY_OVER_SPEED} will be ignored. False by default.*/WEIGHT_MATCHESdiff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.javaindex 37f48aad2d5..1d7f19f4cf2 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/WholeBreakIterator.java@@ -41,7 +41,8 @@ public final class WholeBreakIterator extends BreakIterator {if (pos < start || pos > end) {throw new IllegalArgumentException("offset out of bounds");} else if (pos == end) {- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in+ // something)// https://bugs.openjdk.java.net/browse/JDK-8015110current = end;return DONE;@@ -88,7 +89,8 @@ public final class WholeBreakIterator extends BreakIterator {if (pos < start || pos > end) {throw new IllegalArgumentException("offset out of bounds");} else if (pos == start) {- // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in something)+ // this conflicts with the javadocs, but matches actual behavior (Oracle has a bug in+ // something)// https://bugs.openjdk.java.net/browse/JDK-8015110current = start;return DONE;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/package-info.javaindex b0acd36a42c..4c956760697 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/package-info.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/package-info.java@@ -16,7 +16,9 @@*//**- * The UnifiedHighlighter -- a flexible highlighter that can get offsets from postings, term vectors, or analysis.+ * The UnifiedHighlighter -- a flexible highlighter that can get offsets from postings, term+ * vectors, or analysis.+ ** @see org.apache.lucene.search.uhighlight.UnifiedHighlighter*/package org.apache.lucene.search.uhighlight;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.javaindex 888eeccfa05..c43d497c41a 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragListBuilder.java@@ -19,59 +19,66 @@ package org.apache.lucene.search.vectorhighlight;import java.util.ArrayList;import java.util.Iterator;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;-/**- * A abstract implementation of {@link FragListBuilder}.- */+/** A abstract implementation of {@link FragListBuilder}. */public abstract class BaseFragListBuilder implements FragListBuilder {-+public static final int MARGIN_DEFAULT = 6;public static final int MIN_FRAG_CHAR_SIZE_FACTOR = 3;final int margin;final int minFragCharSize;- public BaseFragListBuilder( int margin ){- if( margin < 0 )- throw new IllegalArgumentException( "margin(" + margin + ") is too small. It must be 0 or higher." );+ public BaseFragListBuilder(int margin) {+ if (margin < 0)+ throw new IllegalArgumentException(+ "margin(" + margin + ") is too small. It must be 0 or higher.");this.margin = margin;- this.minFragCharSize = Math.max( 1, margin * MIN_FRAG_CHAR_SIZE_FACTOR );+ this.minFragCharSize = Math.max(1, margin * MIN_FRAG_CHAR_SIZE_FACTOR);}- public BaseFragListBuilder(){- this( MARGIN_DEFAULT );+ public BaseFragListBuilder() {+ this(MARGIN_DEFAULT);}-- protected FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, FieldFragList fieldFragList, int fragCharSize ){- if( fragCharSize < minFragCharSize )- throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " + minFragCharSize + " or higher." );-++ protected FieldFragList createFieldFragList(+ FieldPhraseList fieldPhraseList, FieldFragList fieldFragList, int fragCharSize) {+ if (fragCharSize < minFragCharSize)+ throw new IllegalArgumentException(+ "fragCharSize("+ + fragCharSize+ + ") is too small. It must be "+ + minFragCharSize+ + " or higher.");+List<WeightedPhraseInfo> wpil = new ArrayList<>();- IteratorQueue<WeightedPhraseInfo> queue = new IteratorQueue<>(fieldPhraseList.getPhraseList().iterator());+ IteratorQueue<WeightedPhraseInfo> queue =+ new IteratorQueue<>(fieldPhraseList.getPhraseList().iterator());WeightedPhraseInfo phraseInfo = null;int startOffset = 0;- while((phraseInfo = queue.top()) != null){+ while ((phraseInfo = queue.top()) != null) {// if the phrase violates the border of previous fragment, discard it and try next phrase- if( phraseInfo.getStartOffset() < startOffset ) {+ if (phraseInfo.getStartOffset() < startOffset) {queue.removeTop();continue;}-+wpil.clear();final int currentPhraseStartOffset = phraseInfo.getStartOffset();int currentPhraseEndOffset = phraseInfo.getEndOffset();int spanStart = Math.max(currentPhraseStartOffset - margin, startOffset);int spanEnd = Math.max(currentPhraseEndOffset, spanStart + fragCharSize);- if (acceptPhrase(queue.removeTop(), currentPhraseEndOffset - currentPhraseStartOffset, fragCharSize)) {+ if (acceptPhrase(+ queue.removeTop(), currentPhraseEndOffset - currentPhraseStartOffset, fragCharSize)) {wpil.add(phraseInfo);}- while((phraseInfo = queue.top()) != null) { // pull until we crossed the current spanEnd+ while ((phraseInfo = queue.top()) != null) { // pull until we crossed the current spanEndif (phraseInfo.getEndOffset() <= spanEnd) {currentPhraseEndOffset = phraseInfo.getEndOffset();- if (acceptPhrase(queue.removeTop(), currentPhraseEndOffset - currentPhraseStartOffset, fragCharSize)) {+ if (acceptPhrase(+ queue.removeTop(), currentPhraseEndOffset - currentPhraseStartOffset, fragCharSize)) {wpil.add(phraseInfo);}} else {@@ -81,62 +88,61 @@ public abstract class BaseFragListBuilder implements FragListBuilder {if (wpil.isEmpty()) {continue;}-+final int matchLen = currentPhraseEndOffset - currentPhraseStartOffset;// now recalculate the start and end position to "center" the result- final int newMargin = Math.max(0, (fragCharSize-matchLen)/2); // matchLen can be > fragCharSize prevent IAOOB here+ // matchLen can be > fragCharSize prevent IAOOB here+ final int newMargin = Math.max(0, (fragCharSize - matchLen) / 2);spanStart = currentPhraseStartOffset - newMargin;if (spanStart < startOffset) {spanStart = startOffset;}// whatever is bigger here we grow this out- spanEnd = spanStart + Math.max(matchLen, fragCharSize);+ spanEnd = spanStart + Math.max(matchLen, fragCharSize);startOffset = spanEnd;fieldFragList.add(spanStart, spanEnd, wpil);}return fieldFragList;}-+/**- * A predicate to decide if the given {@link WeightedPhraseInfo} should be- * accepted as a highlighted phrase or if it should be discarded.- * <p>- * The default implementation discards phrases that are composed of more than one term- * and where the matchLength exceeds the fragment character size.- *+ * A predicate to decide if the given {@link WeightedPhraseInfo} should be accepted as a+ * highlighted phrase or if it should be discarded.+ *+ * <p>The default implementation discards phrases that are composed of more than one term and+ * where the matchLength exceeds the fragment character size.+ ** @param info the phrase info to accept* @param matchLength the match length of the current phrase* @param fragCharSize the configured fragment character size* @return <code>true</code> if this phrase info should be accepted as a highligh phrase*/- protected boolean acceptPhrase(WeightedPhraseInfo info, int matchLength, int fragCharSize) {- return info.getTermsOffsets().size() <= 1 || matchLength <= fragCharSize;- }-- private static final class IteratorQueue<T> {- private final Iterator<T> iter;- private T top;-- public IteratorQueue(Iterator<T> iter) {- this.iter = iter;- T removeTop = removeTop();- assert removeTop == null;- }-- public T top() {- return top;- }-- public T removeTop() {- T currentTop = top;- if (iter.hasNext()) {- top = iter.next();- } else {- top = null;- }- return currentTop;- }-- }-+ protected boolean acceptPhrase(WeightedPhraseInfo info, int matchLength, int fragCharSize) {+ return info.getTermsOffsets().size() <= 1 || matchLength <= fragCharSize;+ }++ private static final class IteratorQueue<T> {+ private final Iterator<T> iter;+ private T top;++ public IteratorQueue(Iterator<T> iter) {+ this.iter = iter;+ T removeTop = removeTop();+ assert removeTop == null;+ }++ public T top() {+ return top;+ }++ public T removeTop() {+ T currentTop = top;+ if (iter.hasNext()) {+ top = iter.next();+ } else {+ top = null;+ }+ return currentTop;+ }+ }}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.javaindex f81d718fe95..4c6ddf3b34d 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java@@ -25,7 +25,6 @@ import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Objects;-import org.apache.lucene.document.Field;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.TextField;@@ -34,98 +33,125 @@ import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.StoredFieldVisitor;import org.apache.lucene.search.highlight.DefaultEncoder;import org.apache.lucene.search.highlight.Encoder;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;+import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;/**- * Base FragmentsBuilder implementation that supports colored pre/post- * tags and multivalued fields.- * <p>- * Uses {@link BoundaryScanner} to determine fragments.+ * Base FragmentsBuilder implementation that supports colored pre/post tags and multivalued fields.+ *+ * <p>Uses {@link BoundaryScanner} to determine fragments.*/public abstract class BaseFragmentsBuilder implements FragmentsBuilder {protected String[] preTags, postTags;public static final String[] COLORED_PRE_TAGS = {- "<b style=\"background:yellow\">", "<b style=\"background:lawngreen\">", "<b style=\"background:aquamarine\">",- "<b style=\"background:magenta\">", "<b style=\"background:palegreen\">", "<b style=\"background:coral\">",- "<b style=\"background:wheat\">", "<b style=\"background:khaki\">", "<b style=\"background:lime\">",- "<b style=\"background:deepskyblue\">", "<b style=\"background:deeppink\">", "<b style=\"background:salmon\">",- "<b style=\"background:peachpuff\">", "<b style=\"background:violet\">", "<b style=\"background:mediumpurple\">",- "<b style=\"background:palegoldenrod\">", "<b style=\"background:darkkhaki\">", "<b style=\"background:springgreen\">",+ "<b style=\"background:yellow\">", "<b style=\"background:lawngreen\">",+ "<b style=\"background:aquamarine\">",+ "<b style=\"background:magenta\">", "<b style=\"background:palegreen\">",+ "<b style=\"background:coral\">",+ "<b style=\"background:wheat\">", "<b style=\"background:khaki\">",+ "<b style=\"background:lime\">",+ "<b style=\"background:deepskyblue\">", "<b style=\"background:deeppink\">",+ "<b style=\"background:salmon\">",+ "<b style=\"background:peachpuff\">", "<b style=\"background:violet\">",+ "<b style=\"background:mediumpurple\">",+ "<b style=\"background:palegoldenrod\">", "<b style=\"background:darkkhaki\">",+ "<b style=\"background:springgreen\">","<b style=\"background:turquoise\">", "<b style=\"background:powderblue\">"};- public static final String[] COLORED_POST_TAGS = { "</b>" };+ public static final String[] COLORED_POST_TAGS = {"</b>"};private char multiValuedSeparator = ' ';private final BoundaryScanner boundaryScanner;private boolean discreteMultiValueHighlighting = false;-- protected BaseFragmentsBuilder(){- this( new String[]{ "<b>" }, new String[]{ "</b>" } );++ protected BaseFragmentsBuilder() {+ this(new String[] {"<b>"}, new String[] {"</b>"});}-- protected BaseFragmentsBuilder( String[] preTags, String[] postTags ){++ protected BaseFragmentsBuilder(String[] preTags, String[] postTags) {this(preTags, postTags, new SimpleBoundaryScanner());}-- protected BaseFragmentsBuilder(BoundaryScanner boundaryScanner){- this( new String[]{ "<b>" }, new String[]{ "</b>" }, boundaryScanner );++ protected BaseFragmentsBuilder(BoundaryScanner boundaryScanner) {+ this(new String[] {"<b>"}, new String[] {"</b>"}, boundaryScanner);}-- protected BaseFragmentsBuilder( String[] preTags, String[] postTags, BoundaryScanner boundaryScanner ){++ protected BaseFragmentsBuilder(+ String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) {this.preTags = preTags;this.postTags = postTags;this.boundaryScanner = boundaryScanner;}-- static Object checkTagsArgument( Object tags ){- if( tags instanceof String ) return tags;- else if( tags instanceof String[] ) return tags;- throw new IllegalArgumentException( "type of preTags/postTags must be a String or String[]" );++ static Object checkTagsArgument(Object tags) {+ if (tags instanceof String) {+ return tags;+ } else if (tags instanceof String[]) {+ return tags;+ }+ throw new IllegalArgumentException("type of preTags/postTags must be a String or String[]");}-- public abstract List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src );++ public abstract List<WeightedFragInfo> getWeightedFragInfoList(List<WeightedFragInfo> src);private static final Encoder NULL_ENCODER = new DefaultEncoder();@Override- public String createFragment( IndexReader reader, int docId,- String fieldName, FieldFragList fieldFragList ) throws IOException {- return createFragment( reader, docId, fieldName, fieldFragList,- preTags, postTags, NULL_ENCODER );+ public String createFragment(+ IndexReader reader, int docId, String fieldName, FieldFragList fieldFragList)+ throws IOException {+ return createFragment(reader, docId, fieldName, fieldFragList, preTags, postTags, NULL_ENCODER);}@Override- public String[] createFragments( IndexReader reader, int docId,- String fieldName, FieldFragList fieldFragList, int maxNumFragments )+ public String[] createFragments(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ int maxNumFragments)throws IOException {- return createFragments( reader, docId, fieldName, fieldFragList, maxNumFragments,- preTags, postTags, NULL_ENCODER );+ return createFragments(+ reader, docId, fieldName, fieldFragList, maxNumFragments, preTags, postTags, NULL_ENCODER);}-+@Override- public String createFragment( IndexReader reader, int docId,- String fieldName, FieldFragList fieldFragList, String[] preTags, String[] postTags,- Encoder encoder ) throws IOException {- String[] fragments = createFragments( reader, docId, fieldName, fieldFragList, 1,- preTags, postTags, encoder );- if( fragments == null || fragments.length == 0 ) return null;+ public String createFragment(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException {+ String[] fragments =+ createFragments(reader, docId, fieldName, fieldFragList, 1, preTags, postTags, encoder);+ if (fragments == null || fragments.length == 0) return null;return fragments[0];}@Override- public String[] createFragments( IndexReader reader, int docId,- String fieldName, FieldFragList fieldFragList, int maxNumFragments,- String[] preTags, String[] postTags, Encoder encoder ) throws IOException {+ public String[] createFragments(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ int maxNumFragments,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException {- if( maxNumFragments < 0 ) {- throw new IllegalArgumentException( "maxNumFragments(" + maxNumFragments + ") must be positive number." );+ if (maxNumFragments < 0) {+ throw new IllegalArgumentException(+ "maxNumFragments(" + maxNumFragments + ") must be positive number.");}List<WeightedFragInfo> fragInfos = fieldFragList.getFragInfos();- Field[] values = getFields( reader, docId, fieldName );- if( values.length == 0 ) {+ Field[] values = getFields(reader, docId, fieldName);+ if (values.length == 0) {return null;}@@ -135,93 +161,119 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {fragInfos = getWeightedFragInfoList(fragInfos);int limitFragments = maxNumFragments < fragInfos.size() ? maxNumFragments : fragInfos.size();- List<String> fragments = new ArrayList<>( limitFragments );+ List<String> fragments = new ArrayList<>(limitFragments);StringBuilder buffer = new StringBuilder();- int[] nextValueIndex = { 0 };- for( int n = 0; n < limitFragments; n++ ){- WeightedFragInfo fragInfo = fragInfos.get( n );- fragments.add( makeFragment( buffer, nextValueIndex, values, fragInfo, preTags, postTags, encoder ) );+ int[] nextValueIndex = {0};+ for (int n = 0; n < limitFragments; n++) {+ WeightedFragInfo fragInfo = fragInfos.get(n);+ fragments.add(+ makeFragment(buffer, nextValueIndex, values, fragInfo, preTags, postTags, encoder));}- return fragments.toArray( new String[fragments.size()] );+ return fragments.toArray(new String[fragments.size()]);}-- protected Field[] getFields( IndexReader reader, int docId, final String fieldName) throws IOException {++ protected Field[] getFields(IndexReader reader, int docId, final String fieldName)+ throws IOException {// according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field???final List<Field> fields = new ArrayList<>();- reader.document(docId, new StoredFieldVisitor() {-- @Override- public void stringField(FieldInfo fieldInfo, String value) {- Objects.requireNonNull(value, "String value should not be null");- FieldType ft = new FieldType(TextField.TYPE_STORED);- ft.setStoreTermVectors(fieldInfo.hasVectors());- fields.add(new Field(fieldInfo.name, value, ft));- }+ reader.document(+ docId,+ new StoredFieldVisitor() {++ @Override+ public void stringField(FieldInfo fieldInfo, String value) {+ Objects.requireNonNull(value, "String value should not be null");+ FieldType ft = new FieldType(TextField.TYPE_STORED);+ ft.setStoreTermVectors(fieldInfo.hasVectors());+ fields.add(new Field(fieldInfo.name, value, ft));+ }- @Override- public Status needsField(FieldInfo fieldInfo) {- return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO;- }- });+ @Override+ public Status needsField(FieldInfo fieldInfo) {+ return fieldInfo.name.equals(fieldName) ? Status.YES : Status.NO;+ }+ });return fields.toArray(new Field[fields.size()]);}- protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo,- String[] preTags, String[] postTags, Encoder encoder ){+ protected String makeFragment(+ StringBuilder buffer,+ int[] index,+ Field[] values,+ WeightedFragInfo fragInfo,+ String[] preTags,+ String[] postTags,+ Encoder encoder) {StringBuilder fragment = new StringBuilder();final int s = fragInfo.getStartOffset();- int[] modifiedStartOffset = { s };- String src = getFragmentSourceMSO( buffer, index, values, s, fragInfo.getEndOffset(), modifiedStartOffset );+ int[] modifiedStartOffset = {s};+ String src =+ getFragmentSourceMSO(+ buffer, index, values, s, fragInfo.getEndOffset(), modifiedStartOffset);int srcIndex = 0;- for( SubInfo subInfo : fragInfo.getSubInfos() ){- for( Toffs to : subInfo.getTermsOffsets() ){+ for (SubInfo subInfo : fragInfo.getSubInfos()) {+ for (Toffs to : subInfo.getTermsOffsets()) {fragment- .append( encoder.encodeText( src.substring( srcIndex, to.getStartOffset() - modifiedStartOffset[0] ) ) )- .append( getPreTag( preTags, subInfo.getSeqnum() ) )- .append( encoder.encodeText( src.substring( to.getStartOffset() - modifiedStartOffset[0], to.getEndOffset() - modifiedStartOffset[0] ) ) )- .append( getPostTag( postTags, subInfo.getSeqnum() ) );+ .append(+ encoder.encodeText(+ src.substring(srcIndex, to.getStartOffset() - modifiedStartOffset[0])))+ .append(getPreTag(preTags, subInfo.getSeqnum()))+ .append(+ encoder.encodeText(+ src.substring(+ to.getStartOffset() - modifiedStartOffset[0],+ to.getEndOffset() - modifiedStartOffset[0])))+ .append(getPostTag(postTags, subInfo.getSeqnum()));srcIndex = to.getEndOffset() - modifiedStartOffset[0];}}- fragment.append( encoder.encodeText( src.substring( srcIndex ) ) );+ fragment.append(encoder.encodeText(src.substring(srcIndex)));return fragment.toString();}- protected String getFragmentSourceMSO( StringBuilder buffer, int[] index, Field[] values,- int startOffset, int endOffset, int[] modifiedStartOffset ){- while( buffer.length() < endOffset && index[0] < values.length ){- buffer.append( values[index[0]++].stringValue() );- buffer.append( getMultiValuedSeparator() );+ protected String getFragmentSourceMSO(+ StringBuilder buffer,+ int[] index,+ Field[] values,+ int startOffset,+ int endOffset,+ int[] modifiedStartOffset) {+ while (buffer.length() < endOffset && index[0] < values.length) {+ buffer.append(values[index[0]++].stringValue());+ buffer.append(getMultiValuedSeparator());}int bufferLength = buffer.length();// we added the multi value char to the last buffer, ignore itif (values[index[0] - 1].fieldType().tokenized()) {bufferLength--;}- int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );- modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );- return buffer.substring( modifiedStartOffset[0], eo );+ int eo =+ bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset(buffer, endOffset);+ modifiedStartOffset[0] = boundaryScanner.findStartOffset(buffer, startOffset);+ return buffer.substring(modifiedStartOffset[0], eo);}-- protected String getFragmentSource( StringBuilder buffer, int[] index, Field[] values,- int startOffset, int endOffset ){- while( buffer.length() < endOffset && index[0] < values.length ){- buffer.append( values[index[0]].stringValue() );- buffer.append( multiValuedSeparator );++ protected String getFragmentSource(+ StringBuilder buffer, int[] index, Field[] values, int startOffset, int endOffset) {+ while (buffer.length() < endOffset && index[0] < values.length) {+ buffer.append(values[index[0]].stringValue());+ buffer.append(multiValuedSeparator);index[0]++;}int eo = buffer.length() < endOffset ? buffer.length() : endOffset;- return buffer.substring( startOffset, eo );+ return buffer.substring(startOffset, eo);}- protected List<WeightedFragInfo> discreteMultiValueHighlighting(List<WeightedFragInfo> fragInfos, Field[] fields) {+ protected List<WeightedFragInfo> discreteMultiValueHighlighting(+ List<WeightedFragInfo> fragInfos, Field[] fields) {Map<String, List<WeightedFragInfo>> fieldNameToFragInfos = new HashMap<>();for (Field field : fields) {fieldNameToFragInfos.put(field.name(), new ArrayList<WeightedFragInfo>());}- fragInfos: for (WeightedFragInfo fragInfo : fragInfos) {+ fragInfos:+ for (WeightedFragInfo fragInfo : fragInfos) {int fieldStart;int fieldEnd = 0;for (Field field : fields) {@@ -232,8 +284,10 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {fieldStart = fieldEnd;fieldEnd += field.stringValue().length() + 1; // + 1 for going to next field with same name.- if (fragInfo.getStartOffset() >= fieldStart && fragInfo.getEndOffset() >= fieldStart &&- fragInfo.getStartOffset() <= fieldEnd && fragInfo.getEndOffset() <= fieldEnd) {+ if (fragInfo.getStartOffset() >= fieldStart+ && fragInfo.getEndOffset() >= fieldStart+ && fragInfo.getStartOffset() <= fieldEnd+ && fragInfo.getEndOffset() <= fieldEnd) {fieldNameToFragInfos.get(field.name()).add(fragInfo);continue fragInfos;}@@ -257,10 +311,10 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {fragEnd = fragInfo.getEndOffset();}-List<SubInfo> subInfos = new ArrayList<>();Iterator<SubInfo> subInfoIterator = fragInfo.getSubInfos().iterator();- float boost = 0.0f; // The boost of the new info will be the sum of the boosts of its SubInfos+ // The boost of the new info will be the sum of the boosts of its SubInfos+ float boost = 0.0f;while (subInfoIterator.hasNext()) {SubInfo subInfo = subInfoIterator.next();List<Toffs> toffsList = new ArrayList<>();@@ -303,7 +357,8 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {}}if (!toffsList.isEmpty()) {- subInfos.add(new SubInfo(subInfo.getText(), toffsList, subInfo.getSeqnum(), subInfo.getBoost()));+ subInfos.add(+ new SubInfo(subInfo.getText(), toffsList, subInfo.getSeqnum(), subInfo.getBoost()));boost += subInfo.getBoost();}@@ -311,7 +366,8 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {subInfoIterator.remove();}}- WeightedFragInfo weightedFragInfo = new WeightedFragInfo(fragStart, fragEnd, subInfos, boost);+ WeightedFragInfo weightedFragInfo =+ new WeightedFragInfo(fragStart, fragEnd, subInfos, boost);fieldNameToFragInfos.get(field.name()).add(weightedFragInfo);}}@@ -320,23 +376,25 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {for (List<WeightedFragInfo> weightedFragInfos : fieldNameToFragInfos.values()) {result.addAll(weightedFragInfos);}- Collections.sort(result, new Comparator<WeightedFragInfo>() {-- @Override- public int compare(FieldFragList.WeightedFragInfo info1, FieldFragList.WeightedFragInfo info2) {- return info1.getStartOffset() - info2.getStartOffset();- }-- });+ Collections.sort(+ result,+ new Comparator<WeightedFragInfo>() {++ @Override+ public int compare(+ FieldFragList.WeightedFragInfo info1, FieldFragList.WeightedFragInfo info2) {+ return info1.getStartOffset() - info2.getStartOffset();+ }+ });return result;}- public void setMultiValuedSeparator( char separator ){+ public void setMultiValuedSeparator(char separator) {multiValuedSeparator = separator;}-- public char getMultiValuedSeparator(){++ public char getMultiValuedSeparator() {return multiValuedSeparator;}@@ -348,20 +406,20 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {this.discreteMultiValueHighlighting = discreteMultiValueHighlighting;}- protected String getPreTag( int num ){- return getPreTag( preTags, num );+ protected String getPreTag(int num) {+ return getPreTag(preTags, num);}-- protected String getPostTag( int num ){- return getPostTag( postTags, num );++ protected String getPostTag(int num) {+ return getPostTag(postTags, num);}-- protected String getPreTag( String[] preTags, int num ){++ protected String getPreTag(String[] preTags, int num) {int n = num % preTags.length;return preTags[n];}-- protected String getPostTag( String[] postTags, int num ){++ protected String getPostTag(String[] postTags, int num) {int n = num % postTags.length;return postTags[n];}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.javaindex 175466189b9..9caace75e43 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java@@ -16,24 +16,24 @@*/package org.apache.lucene.search.vectorhighlight;-/**- * Finds fragment boundaries: pluggable into {@link BaseFragmentsBuilder}- */+/** Finds fragment boundaries: pluggable into {@link BaseFragmentsBuilder} */public interface BoundaryScanner {/*** Scan backward to find end offset.+ ** @param buffer scanned object* @param start start offset to begin* @return the found start offset*/- public int findStartOffset( StringBuilder buffer, int start );+ public int findStartOffset(StringBuilder buffer, int start);/*** Scan forward to find start offset.+ ** @param buffer scanned object* @param start start offset to begin* @return the found end offset*/- public int findEndOffset( StringBuilder buffer, int start );+ public int findEndOffset(StringBuilder buffer, int start);}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.javaindex 1c4042ceea4..8fd9c6dbb1e 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BreakIteratorBoundaryScanner.java@@ -19,22 +19,23 @@ package org.apache.lucene.search.vectorhighlight;import java.text.BreakIterator;/**- * A {@link BoundaryScanner} implementation that uses {@link BreakIterator} to find- * boundaries in the text.+ * A {@link BoundaryScanner} implementation that uses {@link BreakIterator} to find boundaries in+ * the text.+ ** @see BreakIterator*/public class BreakIteratorBoundaryScanner implements BoundaryScanner {-+final BreakIterator bi;- public BreakIteratorBoundaryScanner(BreakIterator bi){+ public BreakIteratorBoundaryScanner(BreakIterator bi) {this.bi = bi;}@Overridepublic int findStartOffset(StringBuilder buffer, int start) {// avoid illegal start offset- if( start > buffer.length() || start < 1 ) return start;+ if (start > buffer.length() || start < 1) return start;bi.setText(buffer.substring(0, start));bi.last();return bi.previous();@@ -43,7 +44,7 @@ public class BreakIteratorBoundaryScanner implements BoundaryScanner {@Overridepublic int findEndOffset(StringBuilder buffer, int start) {// avoid illegal start offset- if( start > buffer.length() || start < 0 ) return start;+ if (start > buffer.length() || start < 0) return start;bi.setText(buffer.substring(start));return bi.next() + start;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.javaindex ac7e9d0a3c4..73997ae7d27 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FastVectorHighlighter.java@@ -19,15 +19,11 @@ package org.apache.lucene.search.vectorhighlight;import java.io.IOException;import java.util.Iterator;import java.util.Set;-import org.apache.lucene.index.IndexReader;import org.apache.lucene.search.Query;import org.apache.lucene.search.highlight.Encoder;-/**- * Another highlighter implementation.- *- */+/** Another highlighter implementation. */public class FastVectorHighlighter {public static final boolean DEFAULT_PHRASE_HIGHLIGHT = true;public static final boolean DEFAULT_FIELD_MATCH = true;@@ -37,33 +33,36 @@ public class FastVectorHighlighter {private final FragmentsBuilder fragmentsBuilder;private int phraseLimit = Integer.MAX_VALUE;- /**- * the default constructor.- */- public FastVectorHighlighter(){- this( DEFAULT_PHRASE_HIGHLIGHT, DEFAULT_FIELD_MATCH );+ /** the default constructor. */+ public FastVectorHighlighter() {+ this(DEFAULT_PHRASE_HIGHLIGHT, DEFAULT_FIELD_MATCH);}/*** a constructor. Using {@link SimpleFragListBuilder} and {@link ScoreOrderFragmentsBuilder}.- *+ ** @param phraseHighlight true or false for phrase highlighting* @param fieldMatch true of false for field matching*/- public FastVectorHighlighter( boolean phraseHighlight, boolean fieldMatch ){- this( phraseHighlight, fieldMatch, new SimpleFragListBuilder(), new ScoreOrderFragmentsBuilder() );+ public FastVectorHighlighter(boolean phraseHighlight, boolean fieldMatch) {+ this(+ phraseHighlight, fieldMatch, new SimpleFragListBuilder(), new ScoreOrderFragmentsBuilder());}/**- * a constructor. A {@link FragListBuilder} and a {@link FragmentsBuilder} can be specified (plugins).- *+ * a constructor. A {@link FragListBuilder} and a {@link FragmentsBuilder} can be specified+ * (plugins).+ ** @param phraseHighlight true of false for phrase highlighting* @param fieldMatch true of false for field matching* @param fragListBuilder an instance of {@link FragListBuilder}* @param fragmentsBuilder an instance of {@link FragmentsBuilder}*/- public FastVectorHighlighter( boolean phraseHighlight, boolean fieldMatch,- FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder ){+ public FastVectorHighlighter(+ boolean phraseHighlight,+ boolean fieldMatch,+ FragListBuilder fragListBuilder,+ FragmentsBuilder fragmentsBuilder) {this.phraseHighlight = phraseHighlight;this.fieldMatch = fieldMatch;this.fragListBuilder = fragListBuilder;@@ -72,34 +71,34 @@ public class FastVectorHighlighter {/*** create a {@link FieldQuery} object.- *+ ** @param query a query* @return the created {@link FieldQuery} object*/- public FieldQuery getFieldQuery( Query query ) {- // TODO: should we deprecate this?+ public FieldQuery getFieldQuery(Query query) {+ // TODO: should we deprecate this?// because if there is no reader, then we cannot rewrite MTQ.try {return getFieldQuery(query, null);} catch (IOException e) {// should never be thrown when reader is null- throw new RuntimeException (e);+ throw new RuntimeException(e);}}-+/*** create a {@link FieldQuery} object.- *+ ** @param query a query* @return the created {@link FieldQuery} object*/- public FieldQuery getFieldQuery( Query query, IndexReader reader ) throws IOException {- return new FieldQuery( query, reader, phraseHighlight, fieldMatch );+ public FieldQuery getFieldQuery(Query query, IndexReader reader) throws IOException {+ return new FieldQuery(query, reader, phraseHighlight, fieldMatch);}/*** return the best fragment.- *+ ** @param fieldQuery {@link FieldQuery} object* @param reader {@link IndexReader} of the index* @param docId document id to be highlighted@@ -108,36 +107,48 @@ public class FastVectorHighlighter {* @return the best fragment (snippet) string* @throws IOException If there is a low-level I/O error*/- public final String getBestFragment( final FieldQuery fieldQuery, IndexReader reader, int docId,- String fieldName, int fragCharSize ) throws IOException {+ public final String getBestFragment(+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String fieldName,+ int fragCharSize)+ throws IOException {FieldFragList fieldFragList =- getFieldFragList( fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize );- return fragmentsBuilder.createFragment( reader, docId, fieldName, fieldFragList );+ getFieldFragList(fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize);+ return fragmentsBuilder.createFragment(reader, docId, fieldName, fieldFragList);}/*** return the best fragments.- *+ ** @param fieldQuery {@link FieldQuery} object* @param reader {@link IndexReader} of the index* @param docId document id to be highlighted* @param fieldName field of the document to be highlighted* @param fragCharSize the length (number of chars) of a fragment* @param maxNumFragments maximum number of fragments- * @return created fragments or null when no fragments created.- * size of the array can be less than maxNumFragments+ * @return created fragments or null when no fragments created. size of the array can be less than+ * maxNumFragments* @throws IOException If there is a low-level I/O error*/- public final String[] getBestFragments( final FieldQuery fieldQuery, IndexReader reader, int docId,- String fieldName, int fragCharSize, int maxNumFragments ) throws IOException {+ public final String[] getBestFragments(+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String fieldName,+ int fragCharSize,+ int maxNumFragments)+ throws IOException {FieldFragList fieldFragList =- getFieldFragList( fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize );- return fragmentsBuilder.createFragments( reader, docId, fieldName, fieldFragList, maxNumFragments );+ getFieldFragList(fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize);+ return fragmentsBuilder.createFragments(+ reader, docId, fieldName, fieldFragList, maxNumFragments);}/*** return the best fragment.- *+ ** @param fieldQuery {@link FieldQuery} object* @param reader {@link IndexReader} of the index* @param docId document id to be highlighted@@ -151,17 +162,27 @@ public class FastVectorHighlighter {* @return the best fragment (snippet) string* @throws IOException If there is a low-level I/O error*/- public final String getBestFragment( final FieldQuery fieldQuery, IndexReader reader, int docId,- String fieldName, int fragCharSize,- FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder,- String[] preTags, String[] postTags, Encoder encoder ) throws IOException {- FieldFragList fieldFragList = getFieldFragList( fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize );- return fragmentsBuilder.createFragment( reader, docId, fieldName, fieldFragList, preTags, postTags, encoder );+ public final String getBestFragment(+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String fieldName,+ int fragCharSize,+ FragListBuilder fragListBuilder,+ FragmentsBuilder fragmentsBuilder,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException {+ FieldFragList fieldFragList =+ getFieldFragList(fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize);+ return fragmentsBuilder.createFragment(+ reader, docId, fieldName, fieldFragList, preTags, postTags, encoder);}/*** return the best fragments.- *+ ** @param fieldQuery {@link FieldQuery} object* @param reader {@link IndexReader} of the index* @param docId document id to be highlighted@@ -173,27 +194,37 @@ public class FastVectorHighlighter {* @param preTags pre-tags to be used to highlight terms* @param postTags post-tags to be used to highlight terms* @param encoder an encoder that generates encoded text- * @return created fragments or null when no fragments created.- * size of the array can be less than maxNumFragments+ * @return created fragments or null when no fragments created. size of the array can be less than+ * maxNumFragments* @throws IOException If there is a low-level I/O error*/- public final String[] getBestFragments( final FieldQuery fieldQuery, IndexReader reader, int docId,- String fieldName, int fragCharSize, int maxNumFragments,- FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder,- String[] preTags, String[] postTags, Encoder encoder ) throws IOException {+ public final String[] getBestFragments(+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String fieldName,+ int fragCharSize,+ int maxNumFragments,+ FragListBuilder fragListBuilder,+ FragmentsBuilder fragmentsBuilder,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException {FieldFragList fieldFragList =- getFieldFragList( fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize );- return fragmentsBuilder.createFragments( reader, docId, fieldName, fieldFragList, maxNumFragments,- preTags, postTags, encoder );+ getFieldFragList(fragListBuilder, fieldQuery, reader, docId, fieldName, fragCharSize);+ return fragmentsBuilder.createFragments(+ reader, docId, fieldName, fieldFragList, maxNumFragments, preTags, postTags, encoder);}/**- * Return the best fragments. Matches are scanned from matchedFields and turned into fragments against- * storedField. The highlighting may not make sense if matchedFields has matches with offsets that don't- * correspond features in storedField. It will outright throw a {@code StringIndexOutOfBoundsException}- * if matchedFields produces offsets outside of storedField. As such it is advisable that all- * matchedFields share the same source as storedField or are at least a prefix of it.- *+ * Return the best fragments. Matches are scanned from matchedFields and turned into fragments+ * against storedField. The highlighting may not make sense if matchedFields has matches with+ * offsets that don't correspond features in storedField. It will outright throw a {@code+ * StringIndexOutOfBoundsException} if matchedFields produces offsets outside of storedField. As+ * such it is advisable that all matchedFields share the same source as storedField or are at+ * least a prefix of it.+ ** @param fieldQuery {@link FieldQuery} object* @param reader {@link IndexReader} of the index* @param docId document id to be highlighted@@ -206,72 +237,96 @@ public class FastVectorHighlighter {* @param preTags pre-tags to be used to highlight terms* @param postTags post-tags to be used to highlight terms* @param encoder an encoder that generates encoded text- * @return created fragments or null when no fragments created.- * size of the array can be less than maxNumFragments+ * @return created fragments or null when no fragments created. size of the array can be less than+ * maxNumFragments* @throws IOException If there is a low-level I/O error*/- public final String[] getBestFragments( final FieldQuery fieldQuery, IndexReader reader, int docId,- String storedField, Set< String > matchedFields, int fragCharSize, int maxNumFragments,- FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder,- String[] preTags, String[] postTags, Encoder encoder ) throws IOException {+ public final String[] getBestFragments(+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String storedField,+ Set<String> matchedFields,+ int fragCharSize,+ int maxNumFragments,+ FragListBuilder fragListBuilder,+ FragmentsBuilder fragmentsBuilder,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException {FieldFragList fieldFragList =- getFieldFragList( fragListBuilder, fieldQuery, reader, docId, matchedFields, fragCharSize );- return fragmentsBuilder.createFragments( reader, docId, storedField, fieldFragList, maxNumFragments,- preTags, postTags, encoder );+ getFieldFragList(fragListBuilder, fieldQuery, reader, docId, matchedFields, fragCharSize);+ return fragmentsBuilder.createFragments(+ reader, docId, storedField, fieldFragList, maxNumFragments, preTags, postTags, encoder);}- /**- * Build a FieldFragList for one field.- */- private FieldFragList getFieldFragList( FragListBuilder fragListBuilder,- final FieldQuery fieldQuery, IndexReader reader, int docId,- String matchedField, int fragCharSize ) throws IOException {- FieldTermStack fieldTermStack = new FieldTermStack( reader, docId, matchedField, fieldQuery );- FieldPhraseList fieldPhraseList = new FieldPhraseList( fieldTermStack, fieldQuery, phraseLimit );- return fragListBuilder.createFieldFragList( fieldPhraseList, fragCharSize );+ /** Build a FieldFragList for one field. */+ private FieldFragList getFieldFragList(+ FragListBuilder fragListBuilder,+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ String matchedField,+ int fragCharSize)+ throws IOException {+ FieldTermStack fieldTermStack = new FieldTermStack(reader, docId, matchedField, fieldQuery);+ FieldPhraseList fieldPhraseList = new FieldPhraseList(fieldTermStack, fieldQuery, phraseLimit);+ return fragListBuilder.createFieldFragList(fieldPhraseList, fragCharSize);}- /**- * Build a FieldFragList for more than one field.- */- private FieldFragList getFieldFragList( FragListBuilder fragListBuilder,- final FieldQuery fieldQuery, IndexReader reader, int docId,- Set< String > matchedFields, int fragCharSize ) throws IOException {- Iterator< String > matchedFieldsItr = matchedFields.iterator();- if ( !matchedFieldsItr.hasNext() ) {- throw new IllegalArgumentException( "matchedFields must contain at least on field name." );+ /** Build a FieldFragList for more than one field. */+ private FieldFragList getFieldFragList(+ FragListBuilder fragListBuilder,+ final FieldQuery fieldQuery,+ IndexReader reader,+ int docId,+ Set<String> matchedFields,+ int fragCharSize)+ throws IOException {+ Iterator<String> matchedFieldsItr = matchedFields.iterator();+ if (!matchedFieldsItr.hasNext()) {+ throw new IllegalArgumentException("matchedFields must contain at least on field name.");}- FieldPhraseList[] toMerge = new FieldPhraseList[ matchedFields.size() ];+ FieldPhraseList[] toMerge = new FieldPhraseList[matchedFields.size()];int i = 0;- while ( matchedFieldsItr.hasNext() ) {- FieldTermStack stack = new FieldTermStack( reader, docId, matchedFieldsItr.next(), fieldQuery );- toMerge[ i++ ] = new FieldPhraseList( stack, fieldQuery, phraseLimit );- }- return fragListBuilder.createFieldFragList( new FieldPhraseList( toMerge ), fragCharSize );+ while (matchedFieldsItr.hasNext()) {+ FieldTermStack stack = new FieldTermStack(reader, docId, matchedFieldsItr.next(), fieldQuery);+ toMerge[i++] = new FieldPhraseList(stack, fieldQuery, phraseLimit);+ }+ return fragListBuilder.createFieldFragList(new FieldPhraseList(toMerge), fragCharSize);}/*** return whether phraseHighlight or not.- *+ ** @return whether phraseHighlight or not*/- public boolean isPhraseHighlight(){ return phraseHighlight; }+ public boolean isPhraseHighlight() {+ return phraseHighlight;+ }/*** return whether fieldMatch or not.- *+ ** @return whether fieldMatch or not*/- public boolean isFieldMatch(){ return fieldMatch; }-+ public boolean isFieldMatch() {+ return fieldMatch;+ }+/*** @return the maximum number of phrases to analyze when searching for the highest-scoring phrase.*/- public int getPhraseLimit () { return phraseLimit; }-+ public int getPhraseLimit() {+ return phraseLimit;+ }+/**- * set the maximum number of phrases to analyze when searching for the highest-scoring phrase.- * The default is unlimited (Integer.MAX_VALUE).+ * set the maximum number of phrases to analyze when searching for the highest-scoring phrase. The+ * default is unlimited (Integer.MAX_VALUE).*/- public void setPhraseLimit (int phraseLimit) { this.phraseLimit = phraseLimit; }+ public void setPhraseLimit(int phraseLimit) {+ this.phraseLimit = phraseLimit;+ }}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.javaindex c4903504f23..d08060b8945 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java@@ -16,15 +16,14 @@*/package org.apache.lucene.search.vectorhighlight;-import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;-import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;-import java.util.ArrayList;import java.util.List;+import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;+import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;/**- * FieldFragList has a list of "frag info" that is used by FragmentsBuilder class- * to create fragments (snippets).+ * FieldFragList has a list of "frag info" that is used by FragmentsBuilder class to create+ * fragments (snippets).*/public abstract class FieldFragList {@@ -32,33 +31,30 @@ public abstract class FieldFragList {/*** a constructor.- *+ ** @param fragCharSize the length (number of chars) of a fragment*/- public FieldFragList( int fragCharSize ){- }+ public FieldFragList(int fragCharSize) {}/*** convert the list of WeightedPhraseInfo to WeightedFragInfo, then add it to the fragInfos- *+ ** @param startOffset start offset of the fragment* @param endOffset end offset of the fragment* @param phraseInfoList list of WeightedPhraseInfo objects*/- public abstract void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList );-+ public abstract void add(int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList);+/*** return the list of WeightedFragInfos.- *+ ** @return fragInfos.- */+ */public List<WeightedFragInfo> getFragInfos() {return fragInfos;}- /**- * List of term offsets + weight for a frag info- */+ /** List of term offsets + weight for a frag info */public static class WeightedFragInfo {private List<SubInfo> subInfos;@@ -66,79 +62,83 @@ public abstract class FieldFragList {private int startOffset;private int endOffset;- public WeightedFragInfo( int startOffset, int endOffset, List<SubInfo> subInfos, float totalBoost ){+ public WeightedFragInfo(+ int startOffset, int endOffset, List<SubInfo> subInfos, float totalBoost) {this.startOffset = startOffset;this.endOffset = endOffset;this.totalBoost = totalBoost;this.subInfos = subInfos;}-- public List<SubInfo> getSubInfos(){++ public List<SubInfo> getSubInfos() {return subInfos;}-- public float getTotalBoost(){++ public float getTotalBoost() {return totalBoost;}-- public int getStartOffset(){++ public int getStartOffset() {return startOffset;}-- public int getEndOffset(){++ public int getEndOffset() {return endOffset;}-+@Override- public String toString(){+ public String toString() {StringBuilder sb = new StringBuilder();- sb.append( "subInfos=(" );- for( SubInfo si : subInfos )- sb.append( si.toString() );- sb.append( ")/" ).append( totalBoost ).append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' );+ sb.append("subInfos=(");+ for (SubInfo si : subInfos) sb.append(si.toString());+ sb.append(")/")+ .append(totalBoost)+ .append('(')+ .append(startOffset)+ .append(',')+ .append(endOffset)+ .append(')');return sb.toString();}-- /**- * Represents the list of term offsets for some text- */++ /** Represents the list of term offsets for some text */public static class SubInfo {- private final String text; // unnecessary member, just exists for debugging purpose- private final List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,- // but if position-gap > 1 and slop > 0 then size() could be greater than 1+ private final String text; // unnecessary member, just exists for debugging purpose+ // usually termsOffsets.size() == 1,+ // but if position-gap > 1 and slop > 0 then size() could be greater than 1+ private final List<Toffs> termsOffsets;private final int seqnum;private final float boost; // used for scoring split WeightedPhraseInfos.- public SubInfo( String text, List<Toffs> termsOffsets, int seqnum, float boost ){+ public SubInfo(String text, List<Toffs> termsOffsets, int seqnum, float boost) {this.text = text;this.termsOffsets = termsOffsets;this.seqnum = seqnum;this.boost = boost;}-- public List<Toffs> getTermsOffsets(){++ public List<Toffs> getTermsOffsets() {return termsOffsets;}-- public int getSeqnum(){++ public int getSeqnum() {return seqnum;}- public String getText(){+ public String getText() {return text;}- public float getBoost(){+ public float getBoost() {return boost;}@Override- public String toString(){+ public String toString() {StringBuilder sb = new StringBuilder();- sb.append( text ).append( '(' );- for( Toffs to : termsOffsets )- sb.append( to.toString() );- sb.append( ')' );+ sb.append(text).append('(');+ for (Toffs to : termsOffsets) sb.append(to.toString());+ sb.append(')');return sb.toString();}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.javaindex 2d9fb2f86d1..e71d3f569c2 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java@@ -15,107 +15,105 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;+import java.util.ArrayList;import java.util.Collection;import java.util.Iterator;import java.util.LinkedList;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;import org.apache.lucene.util.MergedIterator;/**- * FieldPhraseList has a list of WeightedPhraseInfo that is used by FragListBuilder- * to create a FieldFragList object.+ * FieldPhraseList has a list of WeightedPhraseInfo that is used by FragListBuilder to create a+ * FieldFragList object.*/public class FieldPhraseList {- /**- * List of non-overlapping WeightedPhraseInfo objects.- */+ /** List of non-overlapping WeightedPhraseInfo objects. */LinkedList<WeightedPhraseInfo> phraseList = new LinkedList<>();-+/*** create a FieldPhraseList that has no limit on the number of phrases to analyze- *+ ** @param fieldTermStack FieldTermStack object* @param fieldQuery FieldQuery object*/- public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery){- this (fieldTermStack, fieldQuery, Integer.MAX_VALUE);+ public FieldPhraseList(FieldTermStack fieldTermStack, FieldQuery fieldQuery) {+ this(fieldTermStack, fieldQuery, Integer.MAX_VALUE);}-+/*** return the list of WeightedPhraseInfo.- *+ ** @return phraseList.- */+ */public List<WeightedPhraseInfo> getPhraseList() {- return phraseList;+ return phraseList;}-+/*** a constructor.- *+ ** @param fieldTermStack FieldTermStack object* @param fieldQuery FieldQuery object* @param phraseLimit maximum size of phraseList*/- public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit ){+ public FieldPhraseList(FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit) {final String field = fieldTermStack.getFieldName();LinkedList<TermInfo> phraseCandidate = new LinkedList<>();QueryPhraseMap currMap = null;QueryPhraseMap nextMap = null;- while( !fieldTermStack.isEmpty() && (phraseList.size() < phraseLimit) )- {+ while (!fieldTermStack.isEmpty() && (phraseList.size() < phraseLimit)) {phraseCandidate.clear();TermInfo ti = null;TermInfo first = null;-+first = ti = fieldTermStack.pop();- currMap = fieldQuery.getFieldTermMap( field, ti.getText() );+ currMap = fieldQuery.getFieldTermMap(field, ti.getText());while (currMap == null && ti.getNext() != first) {ti = ti.getNext();- currMap = fieldQuery.getFieldTermMap( field, ti.getText() );+ currMap = fieldQuery.getFieldTermMap(field, ti.getText());}// if not found, discard top TermInfo from stack, then try next element- if( currMap == null ) continue;-+ if (currMap == null) continue;+// if found, search the longest phrase- phraseCandidate.add( ti );- while( true ){+ phraseCandidate.add(ti);+ while (true) {first = ti = fieldTermStack.pop();nextMap = null;- if( ti != null ) {- nextMap = currMap.getTermMap( ti.getText() );+ if (ti != null) {+ nextMap = currMap.getTermMap(ti.getText());while (nextMap == null && ti.getNext() != first) {ti = ti.getNext();- nextMap = currMap.getTermMap( ti.getText() );+ nextMap = currMap.getTermMap(ti.getText());}}- if( ti == null || nextMap == null ){- if( ti != null )- fieldTermStack.push( ti );- if( currMap.isValidTermOrPhrase( phraseCandidate ) ){- addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );- }- else{- while( phraseCandidate.size() > 1 ){- fieldTermStack.push( phraseCandidate.removeLast() );- currMap = fieldQuery.searchPhrase( field, phraseCandidate );- if( currMap != null ){- addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );+ if (ti == null || nextMap == null) {+ if (ti != null) fieldTermStack.push(ti);+ if (currMap.isValidTermOrPhrase(phraseCandidate)) {+ addIfNoOverlap(+ new WeightedPhraseInfo(+ phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber()));+ } else {+ while (phraseCandidate.size() > 1) {+ fieldTermStack.push(phraseCandidate.removeLast());+ currMap = fieldQuery.searchPhrase(field, phraseCandidate);+ if (currMap != null) {+ addIfNoOverlap(+ new WeightedPhraseInfo(+ phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber()));break;}}}break;- }- else{- phraseCandidate.add( ti );+ } else {+ phraseCandidate.add(ti);currMap = nextMap;}}@@ -127,230 +125,220 @@ public class FieldPhraseList {** @param toMerge FieldPhraseLists to merge to build this one*/- public FieldPhraseList( FieldPhraseList[] toMerge ) {+ public FieldPhraseList(FieldPhraseList[] toMerge) {// Merge all overlapping WeightedPhraseInfos// Step 1. Sort by startOffset, endOffset, and boost, in that order.- @SuppressWarnings( { "rawtypes", "unchecked" } )- Iterator< WeightedPhraseInfo >[] allInfos = new Iterator[ toMerge.length ];+ @SuppressWarnings({"rawtypes", "unchecked"})+ Iterator<WeightedPhraseInfo>[] allInfos = new Iterator[toMerge.length];int index = 0;- for ( FieldPhraseList fplToMerge : toMerge ) {- allInfos[ index++ ] = fplToMerge.phraseList.iterator();+ for (FieldPhraseList fplToMerge : toMerge) {+ allInfos[index++] = fplToMerge.phraseList.iterator();}- MergedIterator< WeightedPhraseInfo > itr = new MergedIterator<>( false, allInfos );+ MergedIterator<WeightedPhraseInfo> itr = new MergedIterator<>(false, allInfos);// Step 2. Walk the sorted list merging infos that overlapphraseList = new LinkedList<>();- if ( !itr.hasNext() ) {+ if (!itr.hasNext()) {return;}- List< WeightedPhraseInfo > work = new ArrayList<>();+ List<WeightedPhraseInfo> work = new ArrayList<>();WeightedPhraseInfo first = itr.next();- work.add( first );+ work.add(first);int workEndOffset = first.getEndOffset();- while ( itr.hasNext() ) {+ while (itr.hasNext()) {WeightedPhraseInfo current = itr.next();- if ( current.getStartOffset() <= workEndOffset ) {- workEndOffset = Math.max( workEndOffset, current.getEndOffset() );- work.add( current );+ if (current.getStartOffset() <= workEndOffset) {+ workEndOffset = Math.max(workEndOffset, current.getEndOffset());+ work.add(current);} else {- if ( work.size() == 1 ) {- phraseList.add( work.get( 0 ) );- work.set( 0, current );+ if (work.size() == 1) {+ phraseList.add(work.get(0));+ work.set(0, current);} else {- phraseList.add( new WeightedPhraseInfo( work ) );+ phraseList.add(new WeightedPhraseInfo(work));work.clear();- work.add( current );+ work.add(current);}workEndOffset = current.getEndOffset();}}- if ( work.size() == 1 ) {- phraseList.add( work.get( 0 ) );+ if (work.size() == 1) {+ phraseList.add(work.get(0));} else {- phraseList.add( new WeightedPhraseInfo( work ) );+ phraseList.add(new WeightedPhraseInfo(work));work.clear();}}- public void addIfNoOverlap( WeightedPhraseInfo wpi ){- for( WeightedPhraseInfo existWpi : getPhraseList() ){- if( existWpi.isOffsetOverlap( wpi ) ) {- // WeightedPhraseInfo.addIfNoOverlap() dumps the second part of, for example, hyphenated words (social-economics).- // The result is that all informations in TermInfo are lost and not available for further operations.- existWpi.getTermsInfos().addAll( wpi.getTermsInfos() );+ public void addIfNoOverlap(WeightedPhraseInfo wpi) {+ for (WeightedPhraseInfo existWpi : getPhraseList()) {+ if (existWpi.isOffsetOverlap(wpi)) {+ // WeightedPhraseInfo.addIfNoOverlap() dumps the second part of, for example, hyphenated+ // words (social-economics).+ // The result is that all informations in TermInfo are lost and not available for further+ // operations.+ existWpi.getTermsInfos().addAll(wpi.getTermsInfos());return;}}- getPhraseList().add( wpi );+ getPhraseList().add(wpi);}-- /**- * Represents the list of term offsets and boost for some text- */- public static class WeightedPhraseInfo implements Comparable< WeightedPhraseInfo > {- private List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,- // but if position-gap > 1 and slop > 0 then size() could be greater than 1- private float boost; // query boost++ /** Represents the list of term offsets and boost for some text */+ public static class WeightedPhraseInfo implements Comparable<WeightedPhraseInfo> {+ private List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,+ // but if position-gap > 1 and slop > 0 then size() could be greater than 1+ private float boost; // query boostprivate int seqnum;-+private ArrayList<TermInfo> termsInfos;-+/**- * Text of the match, calculated on the fly. Use for debugging only.+ * Text of the match, calculated on the fly. Use for debugging only.+ ** @return the text*/public String getText() {StringBuilder text = new StringBuilder();- for ( TermInfo ti: termsInfos ) {- text.append( ti.getText() );+ for (TermInfo ti : termsInfos) {+ text.append(ti.getText());}return text.toString();}- /**- * @return the termsOffsets- */+ /** @return the termsOffsets */public List<Toffs> getTermsOffsets() {return termsOffsets;}- /**- * @return the boost- */+ /** @return the boost */public float getBoost() {return boost;}- /**- * @return the termInfos- */+ /** @return the termInfos */public List<TermInfo> getTermsInfos() {return termsInfos;}- public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost ){- this( terms, boost, 0 );+ public WeightedPhraseInfo(LinkedList<TermInfo> terms, float boost) {+ this(terms, boost, 0);}-- public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost, int seqnum ){++ public WeightedPhraseInfo(LinkedList<TermInfo> terms, float boost, int seqnum) {this.boost = boost;this.seqnum = seqnum;-+// We keep TermInfos for further operations- termsInfos = new ArrayList<>( terms );-- termsOffsets = new ArrayList<>( terms.size() );- TermInfo ti = terms.get( 0 );- termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) );- if( terms.size() == 1 ){+ termsInfos = new ArrayList<>(terms);++ termsOffsets = new ArrayList<>(terms.size());+ TermInfo ti = terms.get(0);+ termsOffsets.add(new Toffs(ti.getStartOffset(), ti.getEndOffset()));+ if (terms.size() == 1) {return;}int pos = ti.getPosition();- for( int i = 1; i < terms.size(); i++ ){- ti = terms.get( i );- if( ti.getPosition() - pos == 1 ){- Toffs to = termsOffsets.get( termsOffsets.size() - 1 );- to.setEndOffset( ti.getEndOffset() );- }- else{- termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) );+ for (int i = 1; i < terms.size(); i++) {+ ti = terms.get(i);+ if (ti.getPosition() - pos == 1) {+ Toffs to = termsOffsets.get(termsOffsets.size() - 1);+ to.setEndOffset(ti.getEndOffset());+ } else {+ termsOffsets.add(new Toffs(ti.getStartOffset(), ti.getEndOffset()));}pos = ti.getPosition();}}- /**- * Merging constructor. Note that this just grabs seqnum from the first info.- */- public WeightedPhraseInfo( Collection< WeightedPhraseInfo > toMerge ) {+ /** Merging constructor. Note that this just grabs seqnum from the first info. */+ public WeightedPhraseInfo(Collection<WeightedPhraseInfo> toMerge) {// Pretty much the same idea as merging FieldPhraseLists:// Step 1. Sort by startOffset, endOffset// While we are here merge the boosts and termInfos- Iterator< WeightedPhraseInfo > toMergeItr = toMerge.iterator();- if ( !toMergeItr.hasNext() ) {- throw new IllegalArgumentException( "toMerge must contain at least one WeightedPhraseInfo." );+ Iterator<WeightedPhraseInfo> toMergeItr = toMerge.iterator();+ if (!toMergeItr.hasNext()) {+ throw new IllegalArgumentException("toMerge must contain at least one WeightedPhraseInfo.");}WeightedPhraseInfo first = toMergeItr.next();- @SuppressWarnings( { "rawtypes", "unchecked" } )- Iterator< Toffs >[] allToffs = new Iterator[ toMerge.size() ];+ @SuppressWarnings({"rawtypes", "unchecked"})+ Iterator<Toffs>[] allToffs = new Iterator[toMerge.size()];termsInfos = new ArrayList<>();seqnum = first.seqnum;boost = first.boost;- allToffs[ 0 ] = first.termsOffsets.iterator();+ allToffs[0] = first.termsOffsets.iterator();int index = 1;- while ( toMergeItr.hasNext() ) {+ while (toMergeItr.hasNext()) {WeightedPhraseInfo info = toMergeItr.next();boost += info.boost;- termsInfos.addAll( info.termsInfos );- allToffs[ index++ ] = info.termsOffsets.iterator();+ termsInfos.addAll(info.termsInfos);+ allToffs[index++] = info.termsOffsets.iterator();}// Step 2. Walk the sorted list merging overlaps- MergedIterator< Toffs > itr = new MergedIterator<>( false, allToffs );+ MergedIterator<Toffs> itr = new MergedIterator<>(false, allToffs);termsOffsets = new ArrayList<>();- if ( !itr.hasNext() ) {+ if (!itr.hasNext()) {return;}Toffs work = itr.next();- while ( itr.hasNext() ) {+ while (itr.hasNext()) {Toffs current = itr.next();- if ( current.startOffset <= work.endOffset ) {- work.endOffset = Math.max( work.endOffset, current.endOffset );+ if (current.startOffset <= work.endOffset) {+ work.endOffset = Math.max(work.endOffset, current.endOffset);} else {- termsOffsets.add( work );+ termsOffsets.add(work);work = current;}}- termsOffsets.add( work );+ termsOffsets.add(work);}-- public int getStartOffset(){- return termsOffsets.get( 0 ).startOffset;++ public int getStartOffset() {+ return termsOffsets.get(0).startOffset;}-- public int getEndOffset(){- return termsOffsets.get( termsOffsets.size() - 1 ).endOffset;++ public int getEndOffset() {+ return termsOffsets.get(termsOffsets.size() - 1).endOffset;}- public boolean isOffsetOverlap( WeightedPhraseInfo other ){+ public boolean isOffsetOverlap(WeightedPhraseInfo other) {int so = getStartOffset();int eo = getEndOffset();int oso = other.getStartOffset();int oeo = other.getEndOffset();- if( so <= oso && oso < eo ) return true;- if( so < oeo && oeo <= eo ) return true;- if( oso <= so && so < oeo ) return true;- if( oso < eo && eo <= oeo ) return true;+ if (so <= oso && oso < eo) return true;+ if (so < oeo && oeo <= eo) return true;+ if (oso <= so && so < oeo) return true;+ if (oso < eo && eo <= oeo) return true;return false;}-+@Override- public String toString(){+ public String toString() {StringBuilder sb = new StringBuilder();- sb.append( getText() ).append( '(' ).append( boost ).append( ")(" );- for( Toffs to : termsOffsets ){- sb.append( to );+ sb.append(getText()).append('(').append(boost).append(")(");+ for (Toffs to : termsOffsets) {+ sb.append(to);}- sb.append( ')' );+ sb.append(')');return sb.toString();}-- /**- * @return the seqnum- */++ /** @return the seqnum */public int getSeqnum() {return seqnum;}@Override- public int compareTo( WeightedPhraseInfo other ) {+ public int compareTo(WeightedPhraseInfo other) {int diff = getStartOffset() - other.getStartOffset();- if ( diff != 0 ) {+ if (diff != 0) {return diff;}diff = getEndOffset() - other.getEndOffset();- if ( diff != 0 ) {+ if (diff != 0) {return diff;}- return (int) Math.signum( getBoost() - other.getBoost() );+ return (int) Math.signum(getBoost() - other.getBoost());}@Override@@ -359,8 +347,8 @@ public class FieldPhraseList {int result = 1;result = prime * result + getStartOffset();result = prime * result + getEndOffset();- long b = Double.doubleToLongBits( getBoost() );- result = prime * result + ( int )( b ^ ( b >>> 32 ) );+ long b = Double.doubleToLongBits(getBoost());+ result = prime * result + (int) (b ^ (b >>> 32));return result;}@@ -388,33 +376,37 @@ public class FieldPhraseList {return true;}- /**- * Term offsets (start + end)- */- public static class Toffs implements Comparable< Toffs > {+ /** Term offsets (start + end) */+ public static class Toffs implements Comparable<Toffs> {private int startOffset;private int endOffset;- public Toffs( int startOffset, int endOffset ){++ public Toffs(int startOffset, int endOffset) {this.startOffset = startOffset;this.endOffset = endOffset;}- public void setEndOffset( int endOffset ){++ public void setEndOffset(int endOffset) {this.endOffset = endOffset;}- public int getStartOffset(){++ public int getStartOffset() {return startOffset;}- public int getEndOffset(){++ public int getEndOffset() {return endOffset;}+@Override- public int compareTo( Toffs other ) {+ public int compareTo(Toffs other) {int diff = getStartOffset() - other.getStartOffset();- if ( diff != 0 ) {+ if (diff != 0) {return diff;}return getEndOffset() - other.getEndOffset();}+@Overridepublic int hashCode() {final int prime = 31;@@ -423,6 +415,7 @@ public class FieldPhraseList {result = prime * result + getEndOffset();return result;}+@Overridepublic boolean equals(Object obj) {if (this == obj) {@@ -443,10 +436,11 @@ public class FieldPhraseList {}return true;}+@Override- public String toString(){+ public String toString() {StringBuilder sb = new StringBuilder();- sb.append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' );+ sb.append('(').append(startOffset).append(',').append(endOffset).append(')');return sb.toString();}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.javaindex 8584b404bbb..19a51893d1d 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java@@ -15,6 +15,7 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;+import java.io.IOException;import java.util.Collection;import java.util.HashMap;@@ -24,7 +25,6 @@ import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Set;-import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Term;import org.apache.lucene.queries.function.FunctionScoreQuery;@@ -41,8 +41,8 @@ import org.apache.lucene.search.TermQuery;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;/**- * FieldQuery breaks down query object into terms/phrases and keeps- * them in a QueryPhraseMap structure.+ * FieldQuery breaks down query object into terms/phrases and keeps them in a QueryPhraseMap+ * structure.*/public class FieldQuery {@@ -61,75 +61,73 @@ public class FieldQuery {// The maximum number of different matching terms accumulated from any one MultiTermQueryprivate static final int MAX_MTQ_TERMS = 1024;- public FieldQuery(Query query, IndexReader reader, boolean phraseHighlight, boolean fieldMatch) throws IOException {+ public FieldQuery(Query query, IndexReader reader, boolean phraseHighlight, boolean fieldMatch)+ throws IOException {this.fieldMatch = fieldMatch;Set<Query> flatQueries = new LinkedHashSet<>();- flatten( query, reader, flatQueries, 1f );- saveTerms( flatQueries, reader );- Collection<Query> expandQueries = expand( flatQueries );+ flatten(query, reader, flatQueries, 1f);+ saveTerms(flatQueries, reader);+ Collection<Query> expandQueries = expand(flatQueries);- for( Query flatQuery : expandQueries ){- QueryPhraseMap rootMap = getRootMap( flatQuery );- rootMap.add( flatQuery, reader );+ for (Query flatQuery : expandQueries) {+ QueryPhraseMap rootMap = getRootMap(flatQuery);+ rootMap.add(flatQuery, reader);float boost = 1f;while (flatQuery instanceof BoostQuery) {BoostQuery bq = (BoostQuery) flatQuery;flatQuery = bq.getQuery();boost *= bq.getBoost();}- if( !phraseHighlight && flatQuery instanceof PhraseQuery ){- PhraseQuery pq = (PhraseQuery)flatQuery;- if( pq.getTerms().length > 1 ){- for( Term term : pq.getTerms() )- rootMap.addTerm( term, boost );+ if (!phraseHighlight && flatQuery instanceof PhraseQuery) {+ PhraseQuery pq = (PhraseQuery) flatQuery;+ if (pq.getTerms().length > 1) {+ for (Term term : pq.getTerms()) rootMap.addTerm(term, boost);}}}}-- /** For backwards compatibility you can initialize FieldQuery without- * an IndexReader, which is only required to support MultiTermQuery++ /**+ * For backwards compatibility you can initialize FieldQuery without an IndexReader, which is only+ * required to support MultiTermQuery*/- FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ) throws IOException {- this (query, null, phraseHighlight, fieldMatch);+ FieldQuery(Query query, boolean phraseHighlight, boolean fieldMatch) throws IOException {+ this(query, null, phraseHighlight, fieldMatch);}- protected void flatten( Query sourceQuery, IndexReader reader, Collection<Query> flatQueries, float boost ) throws IOException {+ protected void flatten(+ Query sourceQuery, IndexReader reader, Collection<Query> flatQueries, float boost)+ throws IOException {while (sourceQuery instanceof BoostQuery) {BoostQuery bq = (BoostQuery) sourceQuery;sourceQuery = bq.getQuery();boost *= bq.getBoost();}- if( sourceQuery instanceof BooleanQuery ){- BooleanQuery bq = (BooleanQuery)sourceQuery;- for( BooleanClause clause : bq ) {- if( !clause.isProhibited() ) {- flatten( clause.getQuery(), reader, flatQueries, boost );+ if (sourceQuery instanceof BooleanQuery) {+ BooleanQuery bq = (BooleanQuery) sourceQuery;+ for (BooleanClause clause : bq) {+ if (!clause.isProhibited()) {+ flatten(clause.getQuery(), reader, flatQueries, boost);}}- } else if( sourceQuery instanceof DisjunctionMaxQuery ){- DisjunctionMaxQuery dmq = (DisjunctionMaxQuery)sourceQuery;- for( Query query : dmq ){- flatten( query, reader, flatQueries, boost );+ } else if (sourceQuery instanceof DisjunctionMaxQuery) {+ DisjunctionMaxQuery dmq = (DisjunctionMaxQuery) sourceQuery;+ for (Query query : dmq) {+ flatten(query, reader, flatQueries, boost);}- }- else if( sourceQuery instanceof TermQuery ){+ } else if (sourceQuery instanceof TermQuery) {if (boost != 1f) {sourceQuery = new BoostQuery(sourceQuery, boost);}- if( !flatQueries.contains( sourceQuery ) )- flatQueries.add( sourceQuery );- }- else if ( sourceQuery instanceof SynonymQuery ){+ if (!flatQueries.contains(sourceQuery)) flatQueries.add(sourceQuery);+ } else if (sourceQuery instanceof SynonymQuery) {SynonymQuery synQuery = (SynonymQuery) sourceQuery;- for( Term term : synQuery.getTerms()) {- flatten( new TermQuery(term), reader, flatQueries, boost);+ for (Term term : synQuery.getTerms()) {+ flatten(new TermQuery(term), reader, flatQueries, boost);}- }- else if( sourceQuery instanceof PhraseQuery ){- PhraseQuery pq = (PhraseQuery)sourceQuery;- if( pq.getTerms().length == 1 )- sourceQuery = new TermQuery( pq.getTerms()[0] );+ } else if (sourceQuery instanceof PhraseQuery) {+ PhraseQuery pq = (PhraseQuery) sourceQuery;+ if (pq.getTerms().length == 1) sourceQuery = new TermQuery(pq.getTerms()[0]);if (boost != 1f) {sourceQuery = new BoostQuery(sourceQuery, boost);}@@ -137,7 +135,7 @@ public class FieldQuery {} else if (sourceQuery instanceof ConstantScoreQuery) {final Query q = ((ConstantScoreQuery) sourceQuery).getQuery();if (q != null) {- flatten( q, reader, flatQueries, boost);+ flatten(q, reader, flatQueries, boost);}} else if (sourceQuery instanceof FunctionScoreQuery) {final Query q = ((FunctionScoreQuery) sourceQuery).getWrappedQuery();@@ -148,45 +146,47 @@ public class FieldQuery {Query query = sourceQuery;Query rewritten;if (sourceQuery instanceof MultiTermQuery) {- rewritten = new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(MAX_MTQ_TERMS).rewrite(reader, (MultiTermQuery) query);+ rewritten =+ new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(MAX_MTQ_TERMS)+ .rewrite(reader, (MultiTermQuery) query);} else {rewritten = query.rewrite(reader);}if (rewritten != query) {- // only rewrite once and then flatten again - the rewritten query could have a speacial treatment+ // only rewrite once and then flatten again - the rewritten query could have a speacial+ // treatment// if this method is overwritten in a subclass.flatten(rewritten, reader, flatQueries, boost);-- }+ }// if the query is already rewritten we discard it}// else discard queries}-+/** Create expandQueries from flatQueries.- *+ ** expandQueries := flatQueries + overlapped phrase queries- *+ ** ex1) flatQueries={a,b,c}* => expandQueries={a,b,c}* ex2) flatQueries={a,"b c","c d"}* => expandQueries={a,"b c","c d","b c d"}*/- Collection<Query> expand( Collection<Query> flatQueries ){+ Collection<Query> expand(Collection<Query> flatQueries) {Set<Query> expandQueries = new LinkedHashSet<>();- for( Iterator<Query> i = flatQueries.iterator(); i.hasNext(); ){+ for (Iterator<Query> i = flatQueries.iterator(); i.hasNext(); ) {Query query = i.next();i.remove();- expandQueries.add( query );+ expandQueries.add(query);float queryBoost = 1f;while (query instanceof BoostQuery) {BoostQuery bq = (BoostQuery) query;queryBoost *= bq.getBoost();query = bq.getQuery();}- if( !( query instanceof PhraseQuery ) ) continue;- for( Iterator<Query> j = flatQueries.iterator(); j.hasNext(); ){+ if (!(query instanceof PhraseQuery)) continue;+ for (Iterator<Query> j = flatQueries.iterator(); j.hasNext(); ) {Query qj = j.next();float qjBoost = 1f;while (qj instanceof BoostQuery) {@@ -194,8 +194,8 @@ public class FieldQuery {qjBoost *= bq.getBoost();qj = bq.getQuery();}- if( !( qj instanceof PhraseQuery ) ) continue;- checkOverlap( expandQueries, (PhraseQuery)query, queryBoost, (PhraseQuery)qj, qjBoost );+ if (!(qj instanceof PhraseQuery)) continue;+ checkOverlap(expandQueries, (PhraseQuery) query, queryBoost, (PhraseQuery) qj, qjBoost);}}return expandQueries;@@ -203,23 +203,24 @@ public class FieldQuery {/** Check if PhraseQuery A and B have overlapped part.- *+ ** ex1) A="a b", B="b c" => overlap; expandQueries={"a b c"}* ex2) A="b c", B="a b" => overlap; expandQueries={"a b c"}* ex3) A="a b", B="c d" => no overlap; expandQueries={}*/- private void checkOverlap( Collection<Query> expandQueries, PhraseQuery a, float aBoost, PhraseQuery b, float bBoost ){- if( a.getSlop() != b.getSlop() ) return;+ private void checkOverlap(+ Collection<Query> expandQueries, PhraseQuery a, float aBoost, PhraseQuery b, float bBoost) {+ if (a.getSlop() != b.getSlop()) return;Term[] ats = a.getTerms();Term[] bts = b.getTerms();- if( fieldMatch && !ats[0].field().equals( bts[0].field() ) ) return;- checkOverlap( expandQueries, ats, bts, a.getSlop(), aBoost);- checkOverlap( expandQueries, bts, ats, b.getSlop(), bBoost );+ if (fieldMatch && !ats[0].field().equals(bts[0].field())) return;+ checkOverlap(expandQueries, ats, bts, a.getSlop(), aBoost);+ checkOverlap(expandQueries, bts, ats, b.getSlop(), bBoost);}/** Check if src and dest have overlapped part and if it is, create PhraseQueries and add expandQueries.- *+ ** ex1) src="a b", dest="c d" => no overlap* ex2) src="a b", dest="a b c" => no overlap* ex3) src="a b", dest="b c" => overlap; expandQueries={"a b c"}@@ -230,264 +231,244 @@ public class FieldQuery {* expandQueries={"a a a a a","a a a a a a"}* ex8) src="a b c d", dest="b c" => no overlap*/- private void checkOverlap( Collection<Query> expandQueries, Term[] src, Term[] dest, int slop, float boost ){+ private void checkOverlap(+ Collection<Query> expandQueries, Term[] src, Term[] dest, int slop, float boost) {// beginning from 1 (not 0) is safe because that the PhraseQuery has multiple terms// is guaranteed in flatten() method (if PhraseQuery has only one term, flatten()// converts PhraseQuery to TermQuery)- for( int i = 1; i < src.length; i++ ){+ for (int i = 1; i < src.length; i++) {boolean overlap = true;- for( int j = i; j < src.length; j++ ){- if( ( j - i ) < dest.length && !src[j].text().equals( dest[j-i].text() ) ){+ for (int j = i; j < src.length; j++) {+ if ((j - i) < dest.length && !src[j].text().equals(dest[j - i].text())) {overlap = false;break;}}- if( overlap && src.length - i < dest.length ){+ if (overlap && src.length - i < dest.length) {PhraseQuery.Builder pqBuilder = new PhraseQuery.Builder();- for( Term srcTerm : src )- pqBuilder.add( srcTerm );- for( int k = src.length - i; k < dest.length; k++ ){- pqBuilder.add( new Term( src[0].field(), dest[k].text() ) );+ for (Term srcTerm : src) pqBuilder.add(srcTerm);+ for (int k = src.length - i; k < dest.length; k++) {+ pqBuilder.add(new Term(src[0].field(), dest[k].text()));}- pqBuilder.setSlop( slop );+ pqBuilder.setSlop(slop);Query pq = pqBuilder.build();if (boost != 1f) {pq = new BoostQuery(pq, 1f);}- if(!expandQueries.contains( pq ) )- expandQueries.add( pq );+ if (!expandQueries.contains(pq)) expandQueries.add(pq);}}}-- QueryPhraseMap getRootMap( Query query ){- String key = getKey( query );- QueryPhraseMap map = rootMaps.get( key );- if( map == null ){- map = new QueryPhraseMap( this );- rootMaps.put( key, map );++ QueryPhraseMap getRootMap(Query query) {+ String key = getKey(query);+ QueryPhraseMap map = rootMaps.get(key);+ if (map == null) {+ map = new QueryPhraseMap(this);+ rootMaps.put(key, map);}return map;}-+/** Return 'key' string. 'key' is the field name of the Query.* If not fieldMatch, 'key' will be null.*/- private String getKey( Query query ){- if( !fieldMatch ) return null;+ private String getKey(Query query) {+ if (!fieldMatch) return null;while (query instanceof BoostQuery) {query = ((BoostQuery) query).getQuery();}- if( query instanceof TermQuery )- return ((TermQuery)query).getTerm().field();- else if ( query instanceof PhraseQuery ){- PhraseQuery pq = (PhraseQuery)query;+ if (query instanceof TermQuery) return ((TermQuery) query).getTerm().field();+ else if (query instanceof PhraseQuery) {+ PhraseQuery pq = (PhraseQuery) query;Term[] terms = pq.getTerms();return terms[0].field();- }- else if (query instanceof MultiTermQuery) {- return ((MultiTermQuery)query).getField();- }- else- throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );+ } else if (query instanceof MultiTermQuery) {+ return ((MultiTermQuery) query).getField();+ } else throw new RuntimeException("query \"" + query.toString() + "\" must be flatten first.");}/** Save the set of terms in the queries to termSetMap.- *+ ** ex1) q=name:john* - fieldMatch==true* termSetMap=Map<"name",Set<"john">>* - fieldMatch==false* termSetMap=Map<null,Set<"john">>- *+ ** ex2) q=name:john title:manager* - fieldMatch==true* termSetMap=Map<"name",Set<"john">,* "title",Set<"manager">>* - fieldMatch==false* termSetMap=Map<null,Set<"john","manager">>- *+ ** ex3) q=name:"john lennon"* - fieldMatch==true* termSetMap=Map<"name",Set<"john","lennon">>* - fieldMatch==false* termSetMap=Map<null,Set<"john","lennon">>*/- void saveTerms( Collection<Query> flatQueries, IndexReader reader ) throws IOException{- for( Query query : flatQueries ){+ void saveTerms(Collection<Query> flatQueries, IndexReader reader) throws IOException {+ for (Query query : flatQueries) {while (query instanceof BoostQuery) {query = ((BoostQuery) query).getQuery();}- Set<String> termSet = getTermSet( query );- if( query instanceof TermQuery )- termSet.add( ((TermQuery)query).getTerm().text() );- else if( query instanceof PhraseQuery ){- for( Term term : ((PhraseQuery)query).getTerms() )- termSet.add( term.text() );- }- else if (query instanceof MultiTermQuery && reader != null) {+ Set<String> termSet = getTermSet(query);+ if (query instanceof TermQuery) termSet.add(((TermQuery) query).getTerm().text());+ else if (query instanceof PhraseQuery) {+ for (Term term : ((PhraseQuery) query).getTerms()) termSet.add(term.text());+ } else if (query instanceof MultiTermQuery && reader != null) {BooleanQuery mtqTerms = (BooleanQuery) query.rewrite(reader);for (BooleanClause clause : mtqTerms) {- termSet.add (((TermQuery) clause.getQuery()).getTerm().text());+ termSet.add(((TermQuery) clause.getQuery()).getTerm().text());}- }- else- throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );+ } else+ throw new RuntimeException("query \"" + query.toString() + "\" must be flatten first.");}}-- private Set<String> getTermSet( Query query ){- String key = getKey( query );- Set<String> set = termSetMap.get( key );- if( set == null ){++ private Set<String> getTermSet(Query query) {+ String key = getKey(query);+ Set<String> set = termSetMap.get(key);+ if (set == null) {set = new HashSet<>();- termSetMap.put( key, set );+ termSetMap.put(key, set);}return set;}-- Set<String> getTermSet( String field ){- return termSetMap.get( fieldMatch ? field : null );++ Set<String> getTermSet(String field) {+ return termSetMap.get(fieldMatch ? field : null);}- /**- *- * @return QueryPhraseMap- */- public QueryPhraseMap getFieldTermMap( String fieldName, String term ){- QueryPhraseMap rootMap = getRootMap( fieldName );- return rootMap == null ? null : rootMap.subMap.get( term );+ /** @return QueryPhraseMap */+ public QueryPhraseMap getFieldTermMap(String fieldName, String term) {+ QueryPhraseMap rootMap = getRootMap(fieldName);+ return rootMap == null ? null : rootMap.subMap.get(term);}- /**- *- * @return QueryPhraseMap- */- public QueryPhraseMap searchPhrase( String fieldName, final List<TermInfo> phraseCandidate ){- QueryPhraseMap root = getRootMap( fieldName );- if( root == null ) return null;- return root.searchPhrase( phraseCandidate );+ /** @return QueryPhraseMap */+ public QueryPhraseMap searchPhrase(String fieldName, final List<TermInfo> phraseCandidate) {+ QueryPhraseMap root = getRootMap(fieldName);+ if (root == null) return null;+ return root.searchPhrase(phraseCandidate);}-- private QueryPhraseMap getRootMap( String fieldName ){- return rootMaps.get( fieldMatch ? fieldName : null );++ private QueryPhraseMap getRootMap(String fieldName) {+ return rootMaps.get(fieldMatch ? fieldName : null);}-- int nextTermOrPhraseNumber(){++ int nextTermOrPhraseNumber() {return termOrPhraseNumber++;}-- /**- * Internal structure of a query for highlighting: represents- * a nested query structure- */++ /** Internal structure of a query for highlighting: represents a nested query structure */public static class QueryPhraseMap {boolean terminal;- int slop; // valid if terminal == true and phraseHighlight == true- float boost; // valid if terminal == true- int termOrPhraseNumber; // valid if terminal == true+ int slop; // valid if terminal == true and phraseHighlight == true+ float boost; // valid if terminal == true+ int termOrPhraseNumber; // valid if terminal == trueFieldQuery fieldQuery;Map<String, QueryPhraseMap> subMap = new HashMap<>();-- public QueryPhraseMap( FieldQuery fieldQuery ){++ public QueryPhraseMap(FieldQuery fieldQuery) {this.fieldQuery = fieldQuery;}- void addTerm( Term term, float boost ){- QueryPhraseMap map = getOrNewMap( subMap, term.text() );- map.markTerminal( boost );+ void addTerm(Term term, float boost) {+ QueryPhraseMap map = getOrNewMap(subMap, term.text());+ map.markTerminal(boost);}-- private QueryPhraseMap getOrNewMap( Map<String, QueryPhraseMap> subMap, String term ){- QueryPhraseMap map = subMap.get( term );- if( map == null ){- map = new QueryPhraseMap( fieldQuery );- subMap.put( term, map );++ private QueryPhraseMap getOrNewMap(Map<String, QueryPhraseMap> subMap, String term) {+ QueryPhraseMap map = subMap.get(term);+ if (map == null) {+ map = new QueryPhraseMap(fieldQuery);+ subMap.put(term, map);}return map;}- void add( Query query, IndexReader reader ) {+ void add(Query query, IndexReader reader) {float boost = 1f;while (query instanceof BoostQuery) {BoostQuery bq = (BoostQuery) query;query = bq.getQuery();boost = bq.getBoost();}- if( query instanceof TermQuery ){- addTerm( ((TermQuery)query).getTerm(), boost );- }- else if( query instanceof PhraseQuery ){- PhraseQuery pq = (PhraseQuery)query;+ if (query instanceof TermQuery) {+ addTerm(((TermQuery) query).getTerm(), boost);+ } else if (query instanceof PhraseQuery) {+ PhraseQuery pq = (PhraseQuery) query;Term[] terms = pq.getTerms();Map<String, QueryPhraseMap> map = subMap;QueryPhraseMap qpm = null;- for( Term term : terms ){- qpm = getOrNewMap( map, term.text() );+ for (Term term : terms) {+ qpm = getOrNewMap(map, term.text());map = qpm.subMap;}- qpm.markTerminal( pq.getSlop(), boost );- }- else- throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." );+ qpm.markTerminal(pq.getSlop(), boost);+ } else+ throw new RuntimeException("query \"" + query.toString() + "\" must be flatten first.");}-- public QueryPhraseMap getTermMap( String term ){- return subMap.get( term );++ public QueryPhraseMap getTermMap(String term) {+ return subMap.get(term);}-- private void markTerminal( float boost ){- markTerminal( 0, boost );++ private void markTerminal(float boost) {+ markTerminal(0, boost);}-- private void markTerminal( int slop, float boost ){++ private void markTerminal(int slop, float boost) {this.terminal = true;this.slop = slop;this.boost = boost;this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber();}-- public boolean isTerminal(){++ public boolean isTerminal() {return terminal;}-- public int getSlop(){++ public int getSlop() {return slop;}-- public float getBoost(){++ public float getBoost() {return boost;}-- public int getTermOrPhraseNumber(){++ public int getTermOrPhraseNumber() {return termOrPhraseNumber;}-- public QueryPhraseMap searchPhrase( final List<TermInfo> phraseCandidate ){++ public QueryPhraseMap searchPhrase(final List<TermInfo> phraseCandidate) {QueryPhraseMap currMap = this;- for( TermInfo ti : phraseCandidate ){- currMap = currMap.subMap.get( ti.getText() );- if( currMap == null ) return null;+ for (TermInfo ti : phraseCandidate) {+ currMap = currMap.subMap.get(ti.getText());+ if (currMap == null) return null;}- return currMap.isValidTermOrPhrase( phraseCandidate ) ? currMap : null;+ return currMap.isValidTermOrPhrase(phraseCandidate) ? currMap : null;}-- public boolean isValidTermOrPhrase( final List<TermInfo> phraseCandidate ){++ public boolean isValidTermOrPhrase(final List<TermInfo> phraseCandidate) {// check terminal- if( !terminal ) return false;+ if (!terminal) return false;// if the candidate is a term, it is valid- if( phraseCandidate.size() == 1 ) return true;+ if (phraseCandidate.size() == 1) return true;// else check whether the candidate is valid phrase// compare position-gaps between terms to slop- int pos = phraseCandidate.get( 0 ).getPosition();- for( int i = 1; i < phraseCandidate.size(); i++ ){- int nextPos = phraseCandidate.get( i ).getPosition();- if( Math.abs( nextPos - pos - 1 ) > slop ) return false;+ int pos = phraseCandidate.get(0).getPosition();+ for (int i = 1; i < phraseCandidate.size(); i++) {+ int nextPos = phraseCandidate.get(i).getPosition();+ if (Math.abs(nextPos - pos - 1) > slop) return false;pos = nextPos;}return true;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.javaindex 1c9f310aca5..d788febfca1 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java@@ -15,15 +15,15 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;+import java.io.IOException;import java.util.Collections;import java.util.Iterator;import java.util.LinkedList;import java.util.Set;--import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.Fields;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.Term;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;@@ -31,29 +31,31 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRefBuilder;/**- * <code>FieldTermStack</code> is a stack that keeps query terms in the specified field- * of the document to be highlighted.+ * <code>FieldTermStack</code> is a stack that keeps query terms in the specified field of the+ * document to be highlighted.*/public class FieldTermStack {-+private final String fieldName;LinkedList<TermInfo> termList = new LinkedList<>();-+/*** a constructor.- *+ ** @param reader IndexReader of the index* @param docId document id to be highlighted* @param fieldName field of the document to be highlighted* @param fieldQuery FieldQuery object* @throws IOException If there is a low-level I/O error*/- public FieldTermStack( IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery ) throws IOException {+ public FieldTermStack(+ IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery)+ throws IOException {this.fieldName = fieldName;-- Set<String> termSet = fieldQuery.getTermSet( fieldName );++ Set<String> termSet = fieldQuery.getTermSet(fieldName);// just return to make null snippet if un-matched fieldName specified when fieldMatch == true- if( termSet == null ) return;+ if (termSet == null) return;final Fields vectors = reader.getTermVectors(docId);if (vectors == null) {@@ -71,9 +73,9 @@ public class FieldTermStack {final TermsEnum termsEnum = vector.iterator();PostingsEnum dpEnum = null;BytesRef text;-+int numDocs = reader.maxDoc();-+while ((text = termsEnum.next()) != null) {spare.copyUTF8Bytes(text);final String term = spare.toString();@@ -82,24 +84,27 @@ public class FieldTermStack {}dpEnum = termsEnum.postings(dpEnum, PostingsEnum.POSITIONS);dpEnum.nextDoc();-- // For weight look here: http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html- final float weight = ( float ) ( Math.log( numDocs / ( double ) ( reader.docFreq( new Term(fieldName, text) ) + 1 ) ) + 1.0 );++ // For weight look here:+ // http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html+ final float weight =+ (float)+ (Math.log(numDocs / (double) (reader.docFreq(new Term(fieldName, text)) + 1)) + 1.0);final int freq = dpEnum.freq();-- for(int i = 0;i < freq;i++) {++ for (int i = 0; i < freq; i++) {int pos = dpEnum.nextPosition();if (dpEnum.startOffset() < 0) {return; // no offsets, null snippet}- termList.add( new TermInfo( term, dpEnum.startOffset(), dpEnum.endOffset(), pos, weight ) );+ termList.add(new TermInfo(term, dpEnum.startOffset(), dpEnum.endOffset(), pos, weight));}}-+// sort by positionCollections.sort(termList);-+// now look for dups at the same position, linking them togetherint currentPos = -1;TermInfo previous = null;@@ -125,55 +130,49 @@ public class FieldTermStack {}}- /**- * @return field name- */- public String getFieldName(){+ /** @return field name */+ public String getFieldName() {return fieldName;}- /**- * @return the top TermInfo object of the stack- */- public TermInfo pop(){+ /** @return the top TermInfo object of the stack */+ public TermInfo pop() {return termList.poll();}- /**- * @param termInfo the TermInfo object to be put on the top of the stack- */- public void push( TermInfo termInfo ){- termList.push( termInfo );+ /** @param termInfo the TermInfo object to be put on the top of the stack */+ public void push(TermInfo termInfo) {+ termList.push(termInfo);}/*** to know whether the stack is empty- *+ ** @return true if the stack is empty, false if not*/- public boolean isEmpty(){+ public boolean isEmpty() {return termList == null || termList.size() == 0;}-+/**- * Single term with its position/offsets in the document and IDF weight.- * It is Comparable but considers only position.+ * Single term with its position/offsets in the document and IDF weight. It is Comparable but+ * considers only position.*/- public static class TermInfo implements Comparable<TermInfo>{+ public static class TermInfo implements Comparable<TermInfo> {private final String text;private final int startOffset;private final int endOffset;- private final int position;+ private final int position;// IDF-weight of this termprivate final float weight;-+// pointer to other TermInfo's at the same position.// this is a circular list, so with no syns, just points to itselfprivate TermInfo next;- TermInfo(String text, int startOffset, int endOffset, int position, float weight){+ TermInfo(String text, int startOffset, int endOffset, int position, float weight) {this.text = text;this.startOffset = startOffset;this.endOffset = endOffset;@@ -181,28 +180,45 @@ public class FieldTermStack {this.weight = weight;this.next = this;}-- void setNext(TermInfo next) { this.next = next; }- /**- * Returns the next TermInfo at this same position.- * This is a circular list!- */- public TermInfo getNext() { return next; }- public String getText(){ return text; }- public int getStartOffset(){ return startOffset; }- public int getEndOffset(){ return endOffset; }- public int getPosition(){ return position; }- public float getWeight(){ return weight; }-++ void setNext(TermInfo next) {+ this.next = next;+ }+ /** Returns the next TermInfo at this same position. This is a circular list! */+ public TermInfo getNext() {+ return next;+ }++ public String getText() {+ return text;+ }++ public int getStartOffset() {+ return startOffset;+ }++ public int getEndOffset() {+ return endOffset;+ }++ public int getPosition() {+ return position;+ }++ public float getWeight() {+ return weight;+ }+@Override- public String toString(){+ public String toString() {return text + '(' + startOffset + ',' + endOffset + ',' + position + ')';}@Override- public int compareTo( TermInfo o ){- return ( this.position - o.position );+ public int compareTo(TermInfo o) {+ return (this.position - o.position);}+@Overridepublic int hashCode() {final int prime = 31;diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.javaindex ddcdf9dd290..358ba2c16db 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragListBuilder.java@@ -17,17 +17,17 @@package org.apache.lucene.search.vectorhighlight;/**- * FragListBuilder is an interface for FieldFragList builder classes.- * A FragListBuilder class can be plugged in to Highlighter.+ * FragListBuilder is an interface for FieldFragList builder classes. A FragListBuilder class can be+ * plugged in to Highlighter.*/public interface FragListBuilder {/*** create a FieldFragList.- *+ ** @param fieldPhraseList FieldPhraseList object* @param fragCharSize the length (number of chars) of a fragment* @return the created FieldFragList object*/- public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize );+ public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize);}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.javaindex 854b7c616d8..7f7add21a82 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FragmentsBuilder.java@@ -17,20 +17,20 @@package org.apache.lucene.search.vectorhighlight;import java.io.IOException;-import org.apache.lucene.index.IndexReader;import org.apache.lucene.search.highlight.Encoder;/**- * {@link org.apache.lucene.search.vectorhighlight.FragmentsBuilder} is an interface for fragments (snippets) builder classes.- * A {@link org.apache.lucene.search.vectorhighlight.FragmentsBuilder} class can be plugged in to- * {@link org.apache.lucene.search.vectorhighlight.FastVectorHighlighter}.+ * {@link org.apache.lucene.search.vectorhighlight.FragmentsBuilder} is an interface for fragments+ * (snippets) builder classes. A {@link org.apache.lucene.search.vectorhighlight.FragmentsBuilder}+ * class can be plugged in to {@link+ * org.apache.lucene.search.vectorhighlight.FastVectorHighlighter}.*/public interface FragmentsBuilder {/*** create a fragment.- *+ ** @param reader IndexReader of the index* @param docId document id to be highlighted* @param fieldName field of the document to be highlighted@@ -38,27 +38,33 @@ public interface FragmentsBuilder {* @return a created fragment or null when no fragment created* @throws IOException If there is a low-level I/O error*/- public String createFragment( IndexReader reader, int docId, String fieldName,- FieldFragList fieldFragList ) throws IOException;+ public String createFragment(+ IndexReader reader, int docId, String fieldName, FieldFragList fieldFragList)+ throws IOException;/*** create multiple fragments.- *+ ** @param reader IndexReader of the index* @param docId document id to be highlighter* @param fieldName field of the document to be highlighted* @param fieldFragList FieldFragList object* @param maxNumFragments maximum number of fragments- * @return created fragments or null when no fragments created.- * size of the array can be less than maxNumFragments+ * @return created fragments or null when no fragments created. size of the array can be less than+ * maxNumFragments* @throws IOException If there is a low-level I/O error*/- public String[] createFragments( IndexReader reader, int docId, String fieldName,- FieldFragList fieldFragList, int maxNumFragments ) throws IOException;+ public String[] createFragments(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ int maxNumFragments)+ throws IOException;/*** create a fragment.- *+ ** @param reader IndexReader of the index* @param docId document id to be highlighted* @param fieldName field of the document to be highlighted@@ -69,13 +75,19 @@ public interface FragmentsBuilder {* @return a created fragment or null when no fragment created* @throws IOException If there is a low-level I/O error*/- public String createFragment( IndexReader reader, int docId, String fieldName,- FieldFragList fieldFragList, String[] preTags, String[] postTags,- Encoder encoder ) throws IOException;+ public String createFragment(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException;/*** create multiple fragments.- *+ ** @param reader IndexReader of the index* @param docId document id to be highlighter* @param fieldName field of the document to be highlighted@@ -84,11 +96,18 @@ public interface FragmentsBuilder {* @param preTags pre-tags to be used to highlight terms* @param postTags post-tags to be used to highlight terms* @param encoder an encoder that generates encoded text- * @return created fragments or null when no fragments created.- * size of the array can be less than maxNumFragments+ * @return created fragments or null when no fragments created. size of the array can be less than+ * maxNumFragments* @throws IOException If there is a low-level I/O error*/- public String[] createFragments( IndexReader reader, int docId, String fieldName,- FieldFragList fieldFragList, int maxNumFragments, String[] preTags, String[] postTags,- Encoder encoder ) throws IOException;+ public String[] createFragments(+ IndexReader reader,+ int docId,+ String fieldName,+ FieldFragList fieldFragList,+ int maxNumFragments,+ String[] preTags,+ String[] postTags,+ Encoder encoder)+ throws IOException;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.javaindex 8adc114afa1..53d4618b84b 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java@@ -19,62 +19,52 @@ package org.apache.lucene.search.vectorhighlight;import java.util.Collections;import java.util.Comparator;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;-/**- * An implementation of FragmentsBuilder that outputs score-order fragments.- */+/** An implementation of FragmentsBuilder that outputs score-order fragments. */public class ScoreOrderFragmentsBuilder extends BaseFragmentsBuilder {- /**- * a constructor.- */- public ScoreOrderFragmentsBuilder(){+ /** a constructor. */+ public ScoreOrderFragmentsBuilder() {super();}/*** a constructor.- *+ ** @param preTags array of pre-tags for markup terms.* @param postTags array of post-tags for markup terms.*/- public ScoreOrderFragmentsBuilder( String[] preTags, String[] postTags ){- super( preTags, postTags );+ public ScoreOrderFragmentsBuilder(String[] preTags, String[] postTags) {+ super(preTags, postTags);}- public ScoreOrderFragmentsBuilder( BoundaryScanner bs ){- super( bs );+ public ScoreOrderFragmentsBuilder(BoundaryScanner bs) {+ super(bs);}- public ScoreOrderFragmentsBuilder( String[] preTags, String[] postTags, BoundaryScanner bs ){- super( preTags, postTags, bs );+ public ScoreOrderFragmentsBuilder(String[] preTags, String[] postTags, BoundaryScanner bs) {+ super(preTags, postTags, bs);}- /**- * Sort by score the list of WeightedFragInfo- */+ /** Sort by score the list of WeightedFragInfo */@Override- public List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src ) {- Collections.sort( src, new ScoreComparator() );+ public List<WeightedFragInfo> getWeightedFragInfoList(List<WeightedFragInfo> src) {+ Collections.sort(src, new ScoreComparator());return src;}- /**- * Comparator for {@link WeightedFragInfo} by boost, breaking ties- * by offset.- */+ /** Comparator for {@link WeightedFragInfo} by boost, breaking ties by offset. */public static class ScoreComparator implements Comparator<WeightedFragInfo> {@Override- public int compare( WeightedFragInfo o1, WeightedFragInfo o2 ) {- if( o1.getTotalBoost() > o2.getTotalBoost() ) return -1;- else if( o1.getTotalBoost() < o2.getTotalBoost() ) return 1;+ public int compare(WeightedFragInfo o1, WeightedFragInfo o2) {+ if (o1.getTotalBoost() > o2.getTotalBoost()) return -1;+ else if (o1.getTotalBoost() < o2.getTotalBoost()) return 1;// if same score then check startOffset- else{- if( o1.getStartOffset() < o2.getStartOffset() ) return -1;- else if( o1.getStartOffset() > o2.getStartOffset() ) return 1;+ else {+ if (o1.getStartOffset() < o2.getStartOffset()) return -1;+ else if (o1.getStartOffset() > o2.getStartOffset()) return 1;}return 0;}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.javaindex af9dfb6cff7..e26d2f3d5a7 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java@@ -21,36 +21,36 @@ import java.util.HashSet;import java.util.Set;/**- * Simple boundary scanner implementation that divides fragments- * based on a set of separator characters.+ * Simple boundary scanner implementation that divides fragments based on a set of separator+ * characters.*/public class SimpleBoundaryScanner implements BoundaryScanner {-+public static final int DEFAULT_MAX_SCAN = 20;public static final Character[] DEFAULT_BOUNDARY_CHARS = {'.', ',', '!', '?', ' ', '\t', '\n'};protected int maxScan;protected Set<Character> boundaryChars;-- public SimpleBoundaryScanner(){- this( DEFAULT_MAX_SCAN, DEFAULT_BOUNDARY_CHARS );++ public SimpleBoundaryScanner() {+ this(DEFAULT_MAX_SCAN, DEFAULT_BOUNDARY_CHARS);}-- public SimpleBoundaryScanner( int maxScan ){- this( maxScan, DEFAULT_BOUNDARY_CHARS );++ public SimpleBoundaryScanner(int maxScan) {+ this(maxScan, DEFAULT_BOUNDARY_CHARS);}-- public SimpleBoundaryScanner( Character[] boundaryChars ){- this( DEFAULT_MAX_SCAN, boundaryChars );++ public SimpleBoundaryScanner(Character[] boundaryChars) {+ this(DEFAULT_MAX_SCAN, boundaryChars);}-- public SimpleBoundaryScanner( int maxScan, Character[] boundaryChars ){++ public SimpleBoundaryScanner(int maxScan, Character[] boundaryChars) {this.maxScan = maxScan;this.boundaryChars = new HashSet<>();this.boundaryChars.addAll(Arrays.asList(boundaryChars));}-- public SimpleBoundaryScanner( int maxScan, Set<Character> boundaryChars ){++ public SimpleBoundaryScanner(int maxScan, Set<Character> boundaryChars) {this.maxScan = maxScan;this.boundaryChars = boundaryChars;}@@ -58,11 +58,11 @@ public class SimpleBoundaryScanner implements BoundaryScanner {@Overridepublic int findStartOffset(StringBuilder buffer, int start) {// avoid illegal start offset- if( start > buffer.length() || start < 1 ) return start;+ if (start > buffer.length() || start < 1) return start;int offset, count = maxScan;- for( offset = start; offset > 0 && count > 0; count-- ){+ for (offset = start; offset > 0 && count > 0; count--) {// found?- if( boundaryChars.contains( buffer.charAt( offset - 1 ) ) ) return offset;+ if (boundaryChars.contains(buffer.charAt(offset - 1))) return offset;offset--;}// if we scanned up to the start of the text, return it, it's a "boundary"@@ -76,12 +76,12 @@ public class SimpleBoundaryScanner implements BoundaryScanner {@Overridepublic int findEndOffset(StringBuilder buffer, int start) {// avoid illegal start offset- if( start > buffer.length() || start < 0 ) return start;+ if (start > buffer.length() || start < 0) return start;int offset, count = maxScan;- //for( offset = start; offset <= buffer.length() && count > 0; count-- ){- for( offset = start; offset < buffer.length() && count > 0; count-- ){+ // for( offset = start; offset <= buffer.length() && count > 0; count-- ){+ for (offset = start; offset < buffer.length() && count > 0; count--) {// found?- if( boundaryChars.contains( buffer.charAt( offset ) ) ) return offset;+ if (boundaryChars.contains(buffer.charAt(offset))) return offset;offset++;}// not founddiff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.javaindex 878613860ba..7fb0be9b744 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFieldFragList.java@@ -18,36 +18,37 @@ package org.apache.lucene.search.vectorhighlight;import java.util.ArrayList;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;-/**- * A simple implementation of {@link FieldFragList}.- */+/** A simple implementation of {@link FieldFragList}. */public class SimpleFieldFragList extends FieldFragList {/*** a constructor.- *+ ** @param fragCharSize the length (number of chars) of a fragment*/- public SimpleFieldFragList( int fragCharSize ) {- super( fragCharSize );+ public SimpleFieldFragList(int fragCharSize) {+ super(fragCharSize);}/* (non-Javadoc)* @see org.apache.lucene.search.vectorhighlight.FieldFragList#add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList )*/@Override- public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ) {+ public void add(int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList) {float totalBoost = 0;List<SubInfo> subInfos = new ArrayList<>();- for( WeightedPhraseInfo phraseInfo : phraseInfoList ){- subInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(), phraseInfo.getSeqnum(), phraseInfo.getBoost() ) );+ for (WeightedPhraseInfo phraseInfo : phraseInfoList) {+ subInfos.add(+ new SubInfo(+ phraseInfo.getText(),+ phraseInfo.getTermsOffsets(),+ phraseInfo.getSeqnum(),+ phraseInfo.getBoost()));totalBoost += phraseInfo.getBoost();}- getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, totalBoost ) );+ getFragInfos().add(new WeightedFragInfo(startOffset, endOffset, subInfos, totalBoost));}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.javaindex 4681d42b4a4..d9954d4e0be 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragListBuilder.java@@ -16,11 +16,9 @@*/package org.apache.lucene.search.vectorhighlight;-/**- * A simple implementation of {@link FragListBuilder}.- */+/** A simple implementation of {@link FragListBuilder}. */public class SimpleFragListBuilder extends BaseFragListBuilder {-+public SimpleFragListBuilder() {super();}@@ -31,10 +29,10 @@ public class SimpleFragListBuilder extends BaseFragListBuilder {/* (non-Javadoc)* @see org.apache.lucene.search.vectorhighlight.FragListBuilder#createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize)- */+ */@Override- public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){- return createFieldFragList( fieldPhraseList, new SimpleFieldFragList( fragCharSize ), fragCharSize );+ public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {+ return createFieldFragList(+ fieldPhraseList, new SimpleFieldFragList(fragCharSize), fragCharSize);}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.javaindex 5901fc52cb4..171c2daace3 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilder.java@@ -17,45 +17,37 @@package org.apache.lucene.search.vectorhighlight;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;-/**- * A simple implementation of FragmentsBuilder.- *- */+/** A simple implementation of FragmentsBuilder. */public class SimpleFragmentsBuilder extends BaseFragmentsBuilder {- /**- * a constructor.- */+ /** a constructor. */public SimpleFragmentsBuilder() {super();}/*** a constructor.- *+ ** @param preTags array of pre-tags for markup terms.* @param postTags array of post-tags for markup terms.*/- public SimpleFragmentsBuilder( String[] preTags, String[] postTags ) {- super( preTags, postTags );+ public SimpleFragmentsBuilder(String[] preTags, String[] postTags) {+ super(preTags, postTags);}- public SimpleFragmentsBuilder( BoundaryScanner bs ) {- super( bs );+ public SimpleFragmentsBuilder(BoundaryScanner bs) {+ super(bs);}- public SimpleFragmentsBuilder( String[] preTags, String[] postTags, BoundaryScanner bs ) {- super( preTags, postTags, bs );+ public SimpleFragmentsBuilder(String[] preTags, String[] postTags, BoundaryScanner bs) {+ super(preTags, postTags, bs);}- /**- * do nothing. return the source list.- */+ /** do nothing. return the source list. */@Override- public List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src ) {+ public List<WeightedFragInfo> getWeightedFragInfoList(List<WeightedFragInfo> src) {return src;}}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.javaindex e86b3f7ddb1..06ae1ee7775 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SingleFragListBuilder.java@@ -19,14 +19,14 @@ package org.apache.lucene.search.vectorhighlight;import java.util.ArrayList;import java.util.Iterator;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;/**- * An implementation class of {@link FragListBuilder} that generates one {@link WeightedFragInfo} object.- * Typical use case of this class is that you can get an entire field contents- * by using both of this class and {@link SimpleFragmentsBuilder}.<br>+ * An implementation class of {@link FragListBuilder} that generates one {@link WeightedFragInfo}+ * object. Typical use case of this class is that you can get an entire field contents by using both+ * of this class and {@link SimpleFragmentsBuilder}.+ ** <pre class="prettyprint">* FastVectorHighlighter h = new FastVectorHighlighter( true, true,* new SingleFragListBuilder(), new SimpleFragmentsBuilder() );@@ -35,24 +35,21 @@ import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInpublic class SingleFragListBuilder implements FragListBuilder {@Override- public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList,- int fragCharSize) {+ public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {- FieldFragList ffl = new SimpleFieldFragList( fragCharSize );+ FieldFragList ffl = new SimpleFieldFragList(fragCharSize);List<WeightedPhraseInfo> wpil = new ArrayList<>();Iterator<WeightedPhraseInfo> ite = fieldPhraseList.phraseList.iterator();WeightedPhraseInfo phraseInfo = null;- while( true ){- if( !ite.hasNext() ) break;+ while (true) {+ if (!ite.hasNext()) break;phraseInfo = ite.next();- if( phraseInfo == null ) break;+ if (phraseInfo == null) break;- wpil.add( phraseInfo );+ wpil.add(phraseInfo);}- if( wpil.size() > 0 )- ffl.add( 0, Integer.MAX_VALUE, wpil );+ if (wpil.size() > 0) ffl.add(0, Integer.MAX_VALUE, wpil);return ffl;}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.javaindex aa8e9dd13e8..6965ee66117 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFieldFragList.java@@ -19,61 +19,67 @@ package org.apache.lucene.search.vectorhighlight;import java.util.ArrayList;import java.util.HashSet;import java.util.List;-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;-/**- * A weighted implementation of {@link FieldFragList}.- */+/** A weighted implementation of {@link FieldFragList}. */public class WeightedFieldFragList extends FieldFragList {/*** a constructor.- *+ ** @param fragCharSize the length (number of chars) of a fragment*/- public WeightedFieldFragList( int fragCharSize ) {- super( fragCharSize );+ public WeightedFieldFragList(int fragCharSize) {+ super(fragCharSize);}/* (non-Javadoc)* @see org.apache.lucene.search.vectorhighlight.FieldFragList#add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList )- */+ */@Override- public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ) {+ public void add(int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList) {List<SubInfo> tempSubInfos = new ArrayList<>();List<SubInfo> realSubInfos = new ArrayList<>();HashSet<String> distinctTerms = new HashSet<>();int length = 0;- for( WeightedPhraseInfo phraseInfo : phraseInfoList ){+ for (WeightedPhraseInfo phraseInfo : phraseInfoList) {float phraseTotalBoost = 0;- for ( TermInfo ti : phraseInfo.getTermsInfos()) {- if ( distinctTerms.add( ti.getText() ) )+ for (TermInfo ti : phraseInfo.getTermsInfos()) {+ if (distinctTerms.add(ti.getText()))phraseTotalBoost += ti.getWeight() * phraseInfo.getBoost();length++;}- tempSubInfos.add( new SubInfo( phraseInfo.getText(), phraseInfo.getTermsOffsets(),- phraseInfo.getSeqnum(), phraseTotalBoost ) );+ tempSubInfos.add(+ new SubInfo(+ phraseInfo.getText(),+ phraseInfo.getTermsOffsets(),+ phraseInfo.getSeqnum(),+ phraseTotalBoost));}-- // We want that terms per fragment (length) is included into the weight. Otherwise a one-word-query- // would cause an equal weight for all fragments regardless of how much words they contain.- // To avoid that fragments containing a high number of words possibly "outrank" more relevant fragments++ // We want that terms per fragment (length) is included into the weight. Otherwise a+ // one-word-query+ // would cause an equal weight for all fragments regardless of how much words they contain.+ // To avoid that fragments containing a high number of words possibly "outrank" more relevant+ // fragments// we "bend" the length with a standard-normalization a little bit.- float norm = length * ( 1 / (float)Math.sqrt( length ) );+ float norm = length * (1 / (float) Math.sqrt(length));float totalBoost = 0;- for ( SubInfo tempSubInfo : tempSubInfos ) {+ for (SubInfo tempSubInfo : tempSubInfos) {float subInfoBoost = tempSubInfo.getBoost() * norm;- realSubInfos.add( new SubInfo( tempSubInfo.getText(), tempSubInfo.getTermsOffsets(),- tempSubInfo.getSeqnum(), subInfoBoost ));+ realSubInfos.add(+ new SubInfo(+ tempSubInfo.getText(),+ tempSubInfo.getTermsOffsets(),+ tempSubInfo.getSeqnum(),+ subInfoBoost));totalBoost += subInfoBoost;}- getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, realSubInfos, totalBoost ) );+ getFragInfos().add(new WeightedFragInfo(startOffset, endOffset, realSubInfos, totalBoost));}--}\ No newline at end of file+}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.javaindex b109efd8290..02c1066efc6 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/WeightedFragListBuilder.java@@ -16,9 +16,7 @@*/package org.apache.lucene.search.vectorhighlight;-/**- * A weighted implementation of {@link FragListBuilder}.- */+/** A weighted implementation of {@link FragListBuilder}. */public class WeightedFragListBuilder extends BaseFragListBuilder {public WeightedFragListBuilder() {@@ -31,10 +29,10 @@ public class WeightedFragListBuilder extends BaseFragListBuilder {/* (non-Javadoc)* @see org.apache.lucene.search.vectorhighlight.FragListBuilder#createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize)- */+ */@Override- public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){- return createFieldFragList( fieldPhraseList, new WeightedFieldFragList( fragCharSize ), fragCharSize );+ public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {+ return createFieldFragList(+ fieldPhraseList, new WeightedFieldFragList(fragCharSize), fragCharSize);}-}diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package-info.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package-info.javaindex 9718180ec5f..fa659b5d6fd 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package-info.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/package-info.java@@ -17,24 +17,26 @@/*** Another highlighter implementation based on term vectors.- *+ ** <h2>Features</h2>+ ** <ul>- * <li>fast for large docs</li>- * <li>support N-gram fields</li>- * <li>support phrase-unit highlighting with slops</li>- * <li>support multi-term (includes wildcard, range, regexp, etc) queries</li>- * <li>highlight fields need to be stored with Positions and Offsets</li>- * <li>take into account query boost and/or IDF-weight to score fragments</li>- * <li>support colored highlight tags</li>- * <li>pluggable FragListBuilder / FieldFragList</li>- * <li>pluggable FragmentsBuilder</li>+ * <li>fast for large docs+ * <li>support N-gram fields+ * <li>support phrase-unit highlighting with slops+ * <li>support multi-term (includes wildcard, range, regexp, etc) queries+ * <li>highlight fields need to be stored with Positions and Offsets+ * <li>take into account query boost and/or IDF-weight to score fragments+ * <li>support colored highlight tags+ * <li>pluggable FragListBuilder / FieldFragList+ * <li>pluggable FragmentsBuilder* </ul>- *+ ** <h2>Algorithm</h2>- * <p>To explain the algorithm, let's use the following sample text- * (to be highlighted) and user query:</p>- *+ *+ * <p>To explain the algorithm, let's use the following sample text (to be highlighted) and user+ * query:+ ** <table style="border: 1px solid">* <caption>sample document and query</caption>* <tr>@@ -46,12 +48,12 @@* <td>Lucene^2 OR "search library"~1</td>* </tr>* </table>- *- * <p>The user query is a BooleanQuery that consists of TermQuery("Lucene")- * with boost of 2 and PhraseQuery("search library") with slop of 1.</p>- * <p>For your convenience, here is the offsets and positions info of the- * sample text.</p>- *+ *+ * <p>The user query is a BooleanQuery that consists of TermQuery("Lucene") with boost of 2 and+ * PhraseQuery("search library") with slop of 1.+ *+ * <p>For your convenience, here is the offsets and positions info of the sample text.+ ** <pre>* +--------+-----------------------------------+* | | 1111111111222222222233333|@@ -62,42 +64,51 @@* |position|0 1 2 3 4 5 |* +--------*-----------------------------------+* </pre>- *+ ** <h3>Step 1.</h3>- * <p>In Step 1, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap} from the user query.- * <code>QueryPhraseMap</code> consists of the following members:</p>+ *+ * <p>In Step 1, Fast Vector Highlighter generates {@link+ * org.apache.lucene.search.vectorhighlight.FieldQuery.QueryPhraseMap} from the user query. <code>+ * QueryPhraseMap</code> consists of the following members:+ ** <pre class="prettyprint">* public class QueryPhraseMap {* boolean terminal;* int slop; // valid if terminal == true and phraseHighlight == true* float boost; // valid if terminal == true* Map<String, QueryPhraseMap> subMap;- * }+ * }* </pre>- * <p><code>QueryPhraseMap</code> has subMap. The key of the subMap is a term- * text in the user query and the value is a subsequent <code>QueryPhraseMap</code>.- * If the query is a term (not phrase), then the subsequent <code>QueryPhraseMap</code>- * is marked as terminal. If the query is a phrase, then the subsequent <code>QueryPhraseMap</code>- * is not a terminal and it has the next term text in the phrase.</p>- *- * <p>From the sample user query, the following <code>QueryPhraseMap</code>- * will be generated:</p>+ *+ * <p><code>QueryPhraseMap</code> has subMap. The key of the subMap is a term text in the user query+ * and the value is a subsequent <code>QueryPhraseMap</code>. If the query is a term (not phrase),+ * then the subsequent <code>QueryPhraseMap</code> is marked as terminal. If the query is a phrase,+ * then the subsequent <code>QueryPhraseMap</code> is not a terminal and it has the next term text+ * in the phrase.+ *+ * <p>From the sample user query, the following <code>QueryPhraseMap</code> will be generated:+ ** <pre>* QueryPhraseMap* +--------+-+ +-------+-+* |"Lucene"|o+->|boost=2|*| * : terminal* +--------+-+ +-------+-+- *+ ** +--------+-+ +---------+-+ +-------+------+-+* |"search"|o+->|"library"|o+->|boost=1|slop=1|*|* +--------+-+ +---------+-+ +-------+------+-+* </pre>- *+ ** <h3>Step 2.</h3>- * <p>In Step 2, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldTermStack}. Fast Vector Highlighter uses term vector data- * (must be stored {@link org.apache.lucene.document.FieldType#setStoreTermVectorOffsets(boolean)} and {@link org.apache.lucene.document.FieldType#setStoreTermVectorPositions(boolean)})- * to generate it. <code>FieldTermStack</code> keeps the terms in the user query.- * Therefore, in this sample case, Fast Vector Highlighter generates the following <code>FieldTermStack</code>:</p>+ *+ * <p>In Step 2, Fast Vector Highlighter generates {@link+ * org.apache.lucene.search.vectorhighlight.FieldTermStack}. Fast Vector Highlighter uses term+ * vector data (must be stored {@link+ * org.apache.lucene.document.FieldType#setStoreTermVectorOffsets(boolean)} and {@link+ * org.apache.lucene.document.FieldType#setStoreTermVectorPositions(boolean)}) to generate it.+ * <code>FieldTermStack</code> keeps the terms in the user query. Therefore, in this sample case,+ * Fast Vector Highlighter generates the following <code>FieldTermStack</code>:+ ** <pre>* FieldTermStack* +------------------+@@ -109,9 +120,13 @@* +------------------+* where : "termText"(startOffset,endOffset,position)* </pre>+ ** <h3>Step 3.</h3>- * <p>In Step 3, Fast Vector Highlighter generates {@link org.apache.lucene.search.vectorhighlight.FieldPhraseList}- * by reference to <code>QueryPhraseMap</code> and <code>FieldTermStack</code>.</p>+ *+ * <p>In Step 3, Fast Vector Highlighter generates {@link+ * org.apache.lucene.search.vectorhighlight.FieldPhraseList} by reference to <code>QueryPhraseMap+ * </code> and <code>FieldTermStack</code>.+ ** <pre>* FieldPhraseList* +----------------+-----------------+---+@@ -120,13 +135,16 @@* |"search library"|[(12,18),(26,33)]|w=1|* +----------------+-----------------+---+* </pre>- * <p>The type of each entry is <code>WeightedPhraseInfo</code> that consists of- * an array of terms offsets and weight.- * </p>+ *+ * <p>The type of each entry is <code>WeightedPhraseInfo</code> that consists of an array of terms+ * offsets and weight.+ ** <h3>Step 4.</h3>- * <p>In Step 4, Fast Vector Highlighter creates <code>FieldFragList</code> by reference to- * <code>FieldPhraseList</code>. In this sample case, the following- * <code>FieldFragList</code> will be generated:</p>+ *+ * <p>In Step 4, Fast Vector Highlighter creates <code>FieldFragList</code> by reference to <code>+ * FieldPhraseList</code>. In this sample case, the following <code>FieldFragList</code> will be+ * generated:+ ** <pre>* FieldFragList* +---------------------------------+@@ -135,10 +153,10 @@* |totalBoost=3 |* +---------------------------------+* </pre>- *- * <p>- * The calculation for each <code>FieldFragList.WeightedFragInfo.totalBoost</code> (weight)+ *+ * <p>The calculation for each <code>FieldFragList.WeightedFragInfo.totalBoost</code> (weight)* depends on the implementation of <code>FieldFragList.add( ... )</code>:+ ** <pre class="prettyprint">* public void add( int startOffset, int endOffset, List<WeightedPhraseInfo> phraseInfoList ) {* float totalBoost = 0;@@ -149,22 +167,31 @@* }* getFragInfos().add( new WeightedFragInfo( startOffset, endOffset, subInfos, totalBoost ) );* }- *+ ** </pre>- * The used implementation of <code>FieldFragList</code> is noted in <code>BaseFragListBuilder.createFieldFragList( ... )</code>:+ *+ * The used implementation of <code>FieldFragList</code> is noted in <code>+ * BaseFragListBuilder.createFieldFragList( ... )</code>:+ ** <pre class="prettyprint">* public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ){* return createFieldFragList( fieldPhraseList, new SimpleFieldFragList( fragCharSize ), fragCharSize );* }* </pre>- * <p>- * Currently there are basically to approaches available:- * </p>+ *+ * <p>Currently there are basically to approaches available:+ ** <ul>- * <li><code>SimpleFragListBuilder using SimpleFieldFragList</code>: <i>sum-of-boosts</i>-approach. The totalBoost is calculated by summarizing the query-boosts per term. Per default a term is boosted by 1.0</li>- * <li><code>WeightedFragListBuilder using WeightedFieldFragList</code>: <i>sum-of-distinct-weights</i>-approach. The totalBoost is calculated by summarizing the IDF-weights of distinct terms.</li>- * </ul>- * <p>Comparison of the two approaches:</p>+ * <li><code>SimpleFragListBuilder using SimpleFieldFragList</code>:+ * <i>sum-of-boosts</i>-approach. The totalBoost is calculated by summarizing the query-boosts+ * per term. Per default a term is boosted by 1.0+ * <li><code>WeightedFragListBuilder using WeightedFieldFragList</code>:+ * <i>sum-of-distinct-weights</i>-approach. The totalBoost is calculated by summarizing the+ * IDF-weights of distinct terms.+ * </ul>+ *+ * <p>Comparison of the two approaches:+ ** <table style="border: 1px solid">* <caption>* query = das alte testament (The Old Testament)@@ -187,9 +214,10 @@* <tr><td>das</td><td>0.7507678</td><td>1.0</td></tr>* <tr><td>das</td><td>0.7507678</td><td>1.0</td></tr>* </table>- *+ ** <h3>Step 5.</h3>- * <p>In Step 5, by using <code>FieldFragList</code> and the field stored data,- * Fast Vector Highlighter creates highlighted snippets!</p>+ *+ * <p>In Step 5, by using <code>FieldFragList</code> and the field stored data, Fast Vector+ * Highlighter creates highlighted snippets!*/package org.apache.lucene.search.vectorhighlight;diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.javaindex 2c989d10d0a..0cd9519b9c5 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighter.java@@ -16,8 +16,6 @@*/package org.apache.lucene.search.highlight;-import javax.xml.parsers.DocumentBuilder;-import javax.xml.parsers.DocumentBuilderFactory;import java.io.ByteArrayInputStream;import java.io.IOException;import java.nio.charset.StandardCharsets;@@ -29,7 +27,8 @@ import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.StringTokenizer;-+import javax.xml.parsers.DocumentBuilder;+import javax.xml.parsers.DocumentBuilderFactory;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.CachingTokenFilter;@@ -97,10 +96,7 @@ import org.apache.lucene.util.automaton.RegExp;import org.w3c.dom.Element;import org.w3c.dom.NodeList;-/**- * JUnit Test for Highlighter class.- *- */+/** JUnit Test for Highlighter class. */public class TestHighlighter extends BaseTokenStreamTestCase implements Formatter {private IndexReader reader;@@ -112,9 +108,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatteint numHighlights = 0;MockAnalyzer analyzer;TopDocs hits;- FieldType fieldType;//see doc()+ FieldType fieldType; // see doc()final FieldType FIELD_TYPE_TV;+{FieldType fieldType = new FieldType(TextField.TYPE_STORED);fieldType.setStoreTermVectors(true);@@ -126,19 +123,25 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}String[] texts = {- "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",- "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",- "JFK has been shot", "John Kennedy has been shot",- "This text has a typo in referring to Keneddy",- "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets",- "Attribute instances are reused for all tokens of a document. Thus, a TokenStream/-Filter needs to update the appropriate Attribute(s) in incrementToken(). The consumer, commonly the Lucene indexer, consumes the data in the Attributes and then calls incrementToken() again until it retuns false, which indicates that the end of the stream was reached. This means that in each call of incrementToken() a TokenStream/-Filter can safely overwrite the data in the Attribute instances. "+ "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",+ "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",+ "JFK has been shot",+ "John Kennedy has been shot",+ "This text has a typo in referring to Keneddy",+ "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc",+ "y z x y z a b",+ "lets is a the lets is a the lets is a the lets",+ "Attribute instances are reused for all tokens of a document. Thus, a TokenStream/-Filter needs to update the appropriate Attribute(s) in incrementToken(). The consumer, commonly the Lucene indexer, consumes the data in the Attributes and then calls incrementToken() again until it retuns false, which indicates that the end of the stream was reached. This means that in each call of incrementToken() a TokenStream/-Filter can safely overwrite the data in the Attribute instances. "};// Convenience method for succinct tests; doesn't represent "best practice"- private TokenStream getAnyTokenStream(String fieldName, int docId)- throws IOException {- return TokenSources.getTokenStream(fieldName, searcher.getIndexReader().getTermVectors(docId),- searcher.doc(docId).get(fieldName), analyzer, -1);+ private TokenStream getAnyTokenStream(String fieldName, int docId) throws IOException {+ return TokenSources.getTokenStream(+ fieldName,+ searcher.getIndexReader().getTermVectors(docId),+ searcher.doc(docId).get(fieldName),+ analyzer,+ -1);}public void testFunctionScoreQuery() throws Exception {@@ -159,8 +162,9 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteFragmenter fragmenter = new SimpleSpanFragmenter(scorer);highlighter.setTextFragmenter(fragmenter);String fragment = highlighter.getBestFragment(stream, storedField);- assertEquals("Hello this is a piece of text that is <B>very</B> long and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);-+ assertEquals(+ "Hello this is a piece of text that is <B>very</B> long and contains too much preamble and the meat is really here which says kennedy has been shot",+ fragment);}public void testQueryScorerHits() throws Exception {@@ -169,11 +173,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattequery = phraseQuery;searcher = newSearcher(reader);TopDocs hits = searcher.search(query, 10);-+QueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(scorer);-for (int i = 0; i < hits.scoreDocs.length; i++) {final int docId = hits.scoreDocs[i].doc;Document doc = searcher.doc(docId);@@ -190,10 +193,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatteif (VERBOSE) System.out.println(fragment);}}-+public void testHighlightingCommonTermsQuery() throws Exception {CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);- query.add(new Term(FIELD_NAME, "this"));//stop-word+ query.add(new Term(FIELD_NAME, "this")); // stop-wordquery.add(new Term(FIELD_NAME, "long"));query.add(new Term(FIELD_NAME, "very"));@@ -211,7 +214,9 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteFragmenter fragmenter = new SimpleSpanFragmenter(scorer);highlighter.setTextFragmenter(fragmenter);String fragment = highlighter.getBestFragment(stream, storedField);- assertEquals("Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);+ assertEquals(+ "Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot",+ fragment);final int docId1 = hits.scoreDocs[1].doc;doc = searcher.doc(docId1);@@ -220,15 +225,18 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattestream = getAnyTokenStream(FIELD_NAME, docId1);highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));fragment = highlighter.getBestFragment(stream, storedField);- assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);+ assertEquals(+ "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>",+ fragment);}public void testHighlightingSynonymQuery() throws Exception {searcher = newSearcher(reader);- Query query = new SynonymQuery.Builder(FIELD_NAME)- .addTerm(new Term(FIELD_NAME, "jfk"))- .addTerm(new Term(FIELD_NAME, "kennedy"))- .build();+ Query query =+ new SynonymQuery.Builder(FIELD_NAME)+ .addTerm(new Term(FIELD_NAME, "jfk"))+ .addTerm(new Term(FIELD_NAME, "kennedy"))+ .build();QueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(scorer);TokenStream stream = getAnyTokenStream(FIELD_NAME, 2);@@ -244,38 +252,38 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteassertEquals("John <B>Kennedy</B> has been shot", fragment);}- public void testHighlightUnknownQueryAfterRewrite() throws IOException, InvalidTokenOffsetsException {- Query query = new Query() {-- @Override- public Query rewrite(IndexReader reader) throws IOException {- CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);- query.add(new Term(FIELD_NAME, "this"));//stop-word- query.add(new Term(FIELD_NAME, "long"));- query.add(new Term(FIELD_NAME, "very"));- return query;- }-- @Override- public void visit(QueryVisitor visitor) {+ public void testHighlightUnknownQueryAfterRewrite()+ throws IOException, InvalidTokenOffsetsException {+ Query query =+ new Query() {++ @Override+ public Query rewrite(IndexReader reader) throws IOException {+ CommonTermsQuery query = new CommonTermsQuery(Occur.MUST, Occur.SHOULD, 3);+ query.add(new Term(FIELD_NAME, "this")); // stop-word+ query.add(new Term(FIELD_NAME, "long"));+ query.add(new Term(FIELD_NAME, "very"));+ return query;+ }- }+ @Override+ public void visit(QueryVisitor visitor) {}- @Override- public String toString(String field) {- return null;- }+ @Override+ public String toString(String field) {+ return null;+ }- @Override- public int hashCode() {- return System.identityHashCode(this);- }+ @Override+ public int hashCode() {+ return System.identityHashCode(this);+ }- @Override- public boolean equals(Object obj) {- return obj == this;- }- };+ @Override+ public boolean equals(Object obj) {+ return obj == this;+ }+ };searcher = newSearcher(reader);TopDocs hits = searcher.search(query, 10, new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE));@@ -291,7 +299,9 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteFragmenter fragmenter = new SimpleSpanFragmenter(scorer);highlighter.setTextFragmenter(fragmenter);String fragment = highlighter.getBestFragment(stream, storedField);- assertEquals("Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot", fragment);+ assertEquals(+ "Hello this is a piece of text that is <B>very</B> <B>long</B> and contains too much preamble and the meat is really here which says kennedy has been shot",+ fragment);final int docId1 = hits.scoreDocs[1].doc;doc = searcher.doc(docId1);@@ -300,10 +310,11 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattestream = getAnyTokenStream(FIELD_NAME, docId1);highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer));fragment = highlighter.getBestFragment(stream, storedField);- assertEquals("This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>", fragment);-+ assertEquals(+ "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is <B>very</B>",+ fragment);}-+public void testHighlightingWithDefaultField() throws Exception {String s1 = "I call our world Flatland, not because we call it so,";@@ -317,8 +328,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteString expected = "I call our <B>world</B> <B>Flatland</B>, not because we call it so,";String observed = highlightField(q, "SOME_FIELD_NAME", s1);if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);- assertEquals("Query in the default field results in text for *ANY* field being highlighted",- expected, observed);+ assertEquals(+ "Query in the default field results in text for *ANY* field being highlighted",+ expected,+ observed);// Verify that a query against a named field does not result in any// highlighting@@ -332,12 +345,11 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatteif (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);assertEquals("Query in a named field does not result in highlighting when that field isn't in the query",- s1, highlightField(q, FIELD_NAME, s1));+ s1,+ highlightField(q, FIELD_NAME, s1));}-- /**- * This method intended for use with <code>testHighlightingWithDefaultField()</code>- */++ /** This method intended for use with <code>testHighlightingWithDefaultField()</code> */private String highlightField(Query query, String fieldName, String text)throws IOException, InvalidTokenOffsetsException {TokenStream tokenStream = analyzer.tokenStream(fieldName, text);@@ -358,14 +370,14 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);highlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}@@ -374,7 +386,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}// LUCENE-2229- public void testSimpleSpanHighlighterWithStopWordsStraddlingFragmentBoundaries() throws Exception {+ public void testSimpleSpanHighlighterWithStopWordsStraddlingFragmentBoundaries()+ throws Exception {doSearching(new PhraseQuery(FIELD_NAME, "all", "tokens"));int maxNumFragmentsRequired = 1;@@ -389,7 +402,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 36));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);assertTrue("Fragment must be less than 60 characters long", result.length() < 60);@@ -430,7 +444,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatteh.getBestFragment(analyzer, f1, content);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 7);}@@ -446,7 +461,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -455,14 +470,15 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 3);-+numHighlights = 0;builder = new PhraseQuery.Builder();@@ -478,7 +494,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattescorer = new QueryScorer(query, FIELD_NAME);highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -487,14 +503,15 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 4);-+numHighlights = 0;builder = new PhraseQuery.Builder();@@ -510,7 +527,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattescorer = new QueryScorer(query, FIELD_NAME);highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -519,25 +536,27 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 4);-}-+public void testSpanRegexQuery() throws Exception {- query = new SpanOrQuery(new SpanMultiTermQueryWrapper<>(new RegexpQuery(new Term(FIELD_NAME, "ken.*"))));+ query =+ new SpanOrQuery(+ new SpanMultiTermQueryWrapper<>(new RegexpQuery(new Term(FIELD_NAME, "ken.*"))));searcher = newSearcher(reader);hits = searcher.search(query, 100);int maxNumFragmentsRequired = 2;QueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -546,15 +565,16 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}-- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",++ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);}-+public void testRegexQuery() throws Exception {query = new RegexpQuery(new Term(FIELD_NAME, "ken.*"));searcher = newSearcher(reader);@@ -563,7 +583,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -572,15 +592,16 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}-- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",++ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);}-+public void testExternalReader() throws Exception {query = new RegexpQuery(new Term(FIELD_NAME, "ken.*"));searcher = newSearcher(reader);@@ -589,7 +610,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, reader, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -598,16 +619,17 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}-+assertTrue(reader.docFreq(new Term(FIELD_NAME, "hello")) > 0);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);}-+public void testDimensionalRangeQuery() throws Exception {// doesn't currently highlight, but make sure it doesn't cause exception eitherquery = IntPoint.newRangeQuery(NUMERIC_FIELD_NAME, 2, 6);@@ -617,43 +639,47 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {- String text = searcher.doc(hits.scoreDocs[i].doc).getField(NUMERIC_FIELD_NAME).numericValue().toString();+ String text =+ searcher+ .doc(hits.scoreDocs[i].doc)+ .getField(NUMERIC_FIELD_NAME)+ .numericValue()+ .toString();TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);highlighter.setTextFragmenter(new SimpleFragmenter(40));-// String result =- highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,"...");- //if (VERBOSE) System.out.println("\t" + result);+ // String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");+ // if (VERBOSE) System.out.println("\t" + result);}--}-+public void testSimpleQueryScorerPhraseHighlighting2() throws Exception {PhraseQuery phraseQuery = new PhraseQuery(5, FIELD_NAME, "text", "piece", "long");doSearching(phraseQuery);int maxNumFragmentsRequired = 2;- QueryScorer scorer = new QueryScorer(query, FIELD_NAME);- Highlighter highlighter = new Highlighter(this,scorer);+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);+ Highlighter highlighter = new Highlighter(this, scorer);highlighter.setTextFragmenter(new SimpleFragmenter(40));-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);String text = doc.get(FIELD_NAME);TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 6);}@@ -673,15 +699,16 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 3);}}-+public void testSimpleSpanFragmenter() throws Exception {Builder builder = new PhraseQuery.Builder();builder.add(new Term(FIELD_NAME, "piece"), 0);@@ -695,7 +722,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteQueryScorer scorer = new QueryScorer(query, FIELD_NAME);Highlighter highlighter = new Highlighter(this, scorer);-+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -704,10 +731,9 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));- String result = highlighter.getBestFragments(tokenStream, text,- maxNumFragmentsRequired, "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);-}phraseQuery = new PhraseQuery(FIELD_NAME, "been", "shot");@@ -715,7 +741,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormattedoSearching(query);maxNumFragmentsRequired = 2;-+scorer = new QueryScorer(query, FIELD_NAME);highlighter = new Highlighter(this, scorer);@@ -725,13 +751,12 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));- String result = highlighter.getBestFragments(tokenStream, text,- maxNumFragmentsRequired, "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);-}}-+// position sensitive query added after position insensitive querypublic void testPosTermStdTerm() throws Exception {BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();@@ -743,10 +768,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormattedoSearching(booleanQuery.build());int maxNumFragmentsRequired = 2;-+QueryScorer scorer = new QueryScorer(query, FIELD_NAME);- Highlighter highlighter = new Highlighter(this,scorer);-+ Highlighter highlighter = new Highlighter(this, scorer);+for (int i = 0; i < hits.totalHits.value; i++) {final int docId = hits.scoreDocs[i].doc;final Document doc = searcher.doc(docId);@@ -755,11 +780,12 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 4);}}@@ -767,7 +793,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattepublic void testQueryScorerMultiPhraseQueryHighlighting() throws Exception {MultiPhraseQuery.Builder mpqb = new MultiPhraseQuery.Builder();- mpqb.add(new Term[] { new Term(FIELD_NAME, "wordx"), new Term(FIELD_NAME, "wordb") });+ mpqb.add(new Term[] {new Term(FIELD_NAME, "wordx"), new Term(FIELD_NAME, "wordb")});mpqb.add(new Term(FIELD_NAME, "wordy"));doSearching(mpqb.build());@@ -784,8 +810,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte* out-of-order additions, but the Query itself seems to match accurately.*/- mpqb.add(new Term[] { new Term(FIELD_NAME, "wordz") }, 2);- mpqb.add(new Term[] { new Term(FIELD_NAME, "wordx") }, 0);+ mpqb.add(new Term[] {new Term(FIELD_NAME, "wordz")}, 2);+ mpqb.add(new Term[] {new Term(FIELD_NAME, "wordx")}, 0);doSearching(mpqb.build());@@ -796,22 +822,29 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}public void testNearSpanSimpleQuery() throws Exception {- doSearching(new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "beginning")),- new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false));-- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- mode = QUERY;- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- }- };+ doSearching(+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "beginning")),+ new SpanTermQuery(new Term(FIELD_NAME, "kennedy"))+ },+ 3,+ false));++ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ mode = QUERY;+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ }+ };helper.run();- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 2);}@@ -826,8 +859,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteString text = doc.get(FIELD_NAME);TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);}// Not sure we can assert anything here - just running to check we dont@@ -835,170 +868,202 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}public void testSpanHighlighting() throws Exception {- Query query1 = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "wordx")),- new SpanTermQuery(new Term(FIELD_NAME, "wordy")) }, 1, false);- Query query2 = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "wordy")),- new SpanTermQuery(new Term(FIELD_NAME, "wordc")) }, 1, false);+ Query query1 =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "wordx")),+ new SpanTermQuery(new Term(FIELD_NAME, "wordy"))+ },+ 1,+ false);+ Query query2 =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "wordy")),+ new SpanTermQuery(new Term(FIELD_NAME, "wordc"))+ },+ 1,+ false);BooleanQuery.Builder bquery = new BooleanQuery.Builder();bquery.add(query1, Occur.SHOULD);bquery.add(query2, Occur.SHOULD);doSearching(bquery.build());- TestHighlightRunner helper = new TestHighlightRunner() {+ TestHighlightRunner helper =+ new TestHighlightRunner() {- @Override- public void run() throws Exception {- mode = QUERY;- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- }- };+ @Override+ public void run() throws Exception {+ mode = QUERY;+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ }+ };helper.run();- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 7);}public void testNotSpanSimpleQuery() throws Exception {- doSearching(new SpanNotQuery(new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "shot")),- new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false), new SpanTermQuery(- new Term(FIELD_NAME, "john"))));- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- mode = QUERY;- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- }- };+ doSearching(+ new SpanNotQuery(+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "shot")),+ new SpanTermQuery(new Term(FIELD_NAME, "kennedy"))+ },+ 3,+ false),+ new SpanTermQuery(new Term(FIELD_NAME, "john"))));+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ mode = QUERY;+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ }+ };helper.run();- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 4);}public void testGetBestFragmentsSimpleQuery() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();}-- public void testGetBestFragmentsConstantScore() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {- @Override- public void run() throws Exception {- numHighlights = 0;- if (random().nextBoolean()) {- BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add(new ConstantScoreQuery(new TermQuery(- new Term(FIELD_NAME, "kennedy"))), Occur.MUST);- bq.add(new ConstantScoreQuery(new TermQuery(new Term(FIELD_NAME, "kennedy"))), Occur.MUST);- doSearching(bq.build());- } else {- doSearching(new ConstantScoreQuery(new TermQuery(new Term(FIELD_NAME,- "kennedy"))));- }- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);- }- };+ public void testGetBestFragmentsConstantScore() throws Exception {+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ if (random().nextBoolean()) {+ BooleanQuery.Builder bq = new BooleanQuery.Builder();+ bq.add(+ new ConstantScoreQuery(new TermQuery(new Term(FIELD_NAME, "kennedy"))),+ Occur.MUST);+ bq.add(+ new ConstantScoreQuery(new TermQuery(new Term(FIELD_NAME, "kennedy"))),+ Occur.MUST);+ doSearching(bq.build());+ } else {+ doSearching(new ConstantScoreQuery(new TermQuery(new Term(FIELD_NAME, "kennedy"))));+ }+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();}public void testGetFuzzyFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- FuzzyQuery fuzzyQuery = new FuzzyQuery(new Term(FIELD_NAME, "kinnedy"), 2);- fuzzyQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);- doSearching(fuzzyQuery);- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this, true);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ FuzzyQuery fuzzyQuery = new FuzzyQuery(new Term(FIELD_NAME, "kinnedy"), 2);+ fuzzyQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ doSearching(fuzzyQuery);+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this, true);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();}public void testGetWildCardFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k?nnedy"));- wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);- doSearching(wildcardQuery);- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k?nnedy"));+ wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ doSearching(wildcardQuery);+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();}public void testGetMidWildCardFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k*dy"));- wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);- doSearching(wildcardQuery);- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 5);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ WildcardQuery wildcardQuery = new WildcardQuery(new Term(FIELD_NAME, "k*dy"));+ wildcardQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ doSearching(wildcardQuery);+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 5);+ }+ };helper.start();}public void testGetRangeFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {+ TestHighlightRunner helper =+ new TestHighlightRunner() {- @Override- public void run() throws Exception {- numHighlights = 0;-- // Need to explicitly set the QueryParser property to use TermRangeQuery- // rather- // than RangeFilters-- TermRangeQuery rangeQuery = new TermRangeQuery(- FIELD_NAME,- new BytesRef("kannedy"),- new BytesRef("kznnedy"),- true, true);- rangeQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);-- query = rangeQuery;- doSearching(query);-- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 5);- }- };+ @Override+ public void run() throws Exception {+ numHighlights = 0;++ // Need to explicitly set the QueryParser property to use TermRangeQuery+ // rather+ // than RangeFilters++ TermRangeQuery rangeQuery =+ new TermRangeQuery(+ FIELD_NAME, new BytesRef("kannedy"), new BytesRef("kznnedy"), true, true);+ rangeQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);++ query = rangeQuery;+ doSearching(query);++ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 5);+ }+ };helper.start();}@@ -1008,7 +1073,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormattenumHighlights = 0;query = new WildcardQuery(new Term(FIELD_NAME, "ken*"));- ((WildcardQuery)query).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);+ ((WildcardQuery) query).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);searcher = newSearcher(reader);// can't rewrite ConstantScore if you want to highlight it -// it rewrites to ConstantScoreQuery which cannot be highlighted@@ -1029,17 +1094,19 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(20));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- fragmentSeparator);+ String result =+ highlighter.getBestFragments(+ tokenStream, text, maxNumFragmentsRequired, fragmentSeparator);if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);-+// try null field-+hits = searcher.search(query, 1000);-+numHighlights = 0;for (int i = 0; i < hits.totalHits.value; i++) {@@ -1055,17 +1122,19 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(20));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- fragmentSeparator);+ String result =+ highlighter.getBestFragments(+ tokenStream, text, maxNumFragmentsRequired, fragmentSeparator);if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);-+// try default field-+hits = searcher.search(query, 1000);-+numHighlights = 0;for (int i = 0; i < hits.totalHits.value; i++) {@@ -1081,374 +1150,411 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(20));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- fragmentSeparator);+ String result =+ highlighter.getBestFragments(+ tokenStream, text, maxNumFragmentsRequired, fragmentSeparator);if (VERBOSE) System.out.println("\t" + result);}- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == 5);}public void testGetBestFragmentsPhrase() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- PhraseQuery phraseQuery = new PhraseQuery(FIELD_NAME, "john", "kennedy");- doSearching(phraseQuery);- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- // Currently highlights "John" and "Kennedy" separately- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 2);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ PhraseQuery phraseQuery = new PhraseQuery(FIELD_NAME, "john", "kennedy");+ doSearching(phraseQuery);+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ // Currently highlights "John" and "Kennedy" separately+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 2);+ }+ };helper.start();}public void testGetBestFragmentsQueryScorer() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),- new SpanTermQuery(new Term("contents", "kennedy")), };-- SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);- doSearching(snq);- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- // Currently highlights "John" and "Kennedy" separately- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 2);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ SpanQuery clauses[] = {+ new SpanTermQuery(new Term("contents", "john")),+ new SpanTermQuery(new Term("contents", "kennedy")),+ };++ SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);+ doSearching(snq);+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ // Currently highlights "John" and "Kennedy" separately+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 2);+ }+ };helper.start();}public void testOffByOne() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- TermQuery query = new TermQuery(new Term("data", "help"));- Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(query));- hg.setTextFragmenter(new NullFragmenter());+ TestHighlightRunner helper =+ new TestHighlightRunner() {- String match = hg.getBestFragment(analyzer, "data", "help me [54-65]");- assertEquals("<B>help</B> me [54-65]", match);+ @Override+ public void run() throws Exception {+ TermQuery query = new TermQuery(new Term("data", "help"));+ Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(query));+ hg.setTextFragmenter(new NullFragmenter());- }- };+ String match = hg.getBestFragment(analyzer, "data", "help me [54-65]");+ assertEquals("<B>help</B> me [54-65]", match);+ }+ };helper.start();}public void testGetBestFragmentsFilteredQuery() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),- new SpanTermQuery(new Term("contents", "kennedy")), };- SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);- BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add(snq, Occur.MUST);- bq.add(TermRangeQuery.newStringRange("contents", "john", "john", true, true), Occur.FILTER);-- doSearching(bq.build());- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- // Currently highlights "John" and "Kennedy" separately- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 2);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ SpanQuery clauses[] = {+ new SpanTermQuery(new Term("contents", "john")),+ new SpanTermQuery(new Term("contents", "kennedy")),+ };+ SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);+ BooleanQuery.Builder bq = new BooleanQuery.Builder();+ bq.add(snq, Occur.MUST);+ bq.add(+ TermRangeQuery.newStringRange("contents", "john", "john", true, true),+ Occur.FILTER);++ doSearching(bq.build());+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ // Currently highlights "John" and "Kennedy" separately+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 2);+ }+ };helper.start();}public void testGetBestFragmentsFilteredPhraseQuery() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- PhraseQuery pq = new PhraseQuery("contents", "john", "kennedy");- BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add(pq, Occur.MUST);- bq.add(TermRangeQuery.newStringRange("contents", "john", "john", true, true), Occur.FILTER);-- doSearching(bq.build());- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- // Currently highlights "John" and "Kennedy" separately- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 2);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ PhraseQuery pq = new PhraseQuery("contents", "john", "kennedy");+ BooleanQuery.Builder bq = new BooleanQuery.Builder();+ bq.add(pq, Occur.MUST);+ bq.add(+ TermRangeQuery.newStringRange("contents", "john", "john", true, true),+ Occur.FILTER);++ doSearching(bq.build());+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ // Currently highlights "John" and "Kennedy" separately+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 2);+ }+ };helper.start();}public void testGetBestFragmentsMultiTerm() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();- booleanQuery.add(new TermQuery(new Term(FIELD_NAME, "john")), Occur.SHOULD);- PrefixQuery prefixQuery = new PrefixQuery(new Term(FIELD_NAME, "kenn"));- prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);- booleanQuery.add(prefixQuery, Occur.SHOULD);-- doSearching(booleanQuery.build());- doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 5);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();+ booleanQuery.add(new TermQuery(new Term(FIELD_NAME, "john")), Occur.SHOULD);+ PrefixQuery prefixQuery = new PrefixQuery(new Term(FIELD_NAME, "kenn"));+ prefixQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);+ booleanQuery.add(prefixQuery, Occur.SHOULD);++ doSearching(booleanQuery.build());+ doStandardHighlights(analyzer, searcher, hits, query, TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 5);+ }+ };helper.start();}public void testGetBestFragmentsWithOr() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;-- BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add(new TermQuery(new Term(FIELD_NAME, "jfk")), Occur.SHOULD);- query.add(new TermQuery(new Term(FIELD_NAME, "kennedy")), Occur.SHOULD);-- doSearching(query.build());- doStandardHighlights(analyzer, searcher, hits, query.build(), TestHighlighter.this);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 5);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;++ BooleanQuery.Builder query = new BooleanQuery.Builder();+ query.add(new TermQuery(new Term(FIELD_NAME, "jfk")), Occur.SHOULD);+ query.add(new TermQuery(new Term(FIELD_NAME, "kennedy")), Occur.SHOULD);++ doSearching(query.build());+ doStandardHighlights(analyzer, searcher, hits, query.build(), TestHighlighter.this);+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 5);+ }+ };helper.start();}public void testGetBestSingleFragment() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));- numHighlights = 0;- for (int i = 0; i < hits.totalHits.value; i++) {- final int docId = hits.scoreDocs[i].doc;- final Document doc = searcher.doc(docId);- String text = doc.get(FIELD_NAME);- TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);-- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);- highlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragment(tokenStream, text);- if (VERBOSE) System.out.println("\t" + result);- }- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);-- numHighlights = 0;- for (int i = 0; i < hits.totalHits.value; i++) {- final int docId = hits.scoreDocs[i].doc;- final Document doc = searcher.doc(docId);- String text = doc.get(FIELD_NAME);- TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);- highlighter.getBestFragment(tokenStream, text);- }- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);-- numHighlights = 0;- for (int i = 0; i < hits.totalHits.value; i++) {- final int docId = hits.scoreDocs[i].doc;- final Document doc = searcher.doc(docId);- String text = doc.get(FIELD_NAME);- TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);-- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);- highlighter.getBestFragments(tokenStream, text, 10);- }- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);-- }-- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));+ numHighlights = 0;+ for (int i = 0; i < hits.totalHits.value; i++) {+ final int docId = hits.scoreDocs[i].doc;+ final Document doc = searcher.doc(docId);+ String text = doc.get(FIELD_NAME);+ TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);++ Highlighter highlighter = getHighlighter(query, FIELD_NAME, TestHighlighter.this);+ highlighter.setTextFragmenter(new SimpleFragmenter(40));+ String result = highlighter.getBestFragment(tokenStream, text);+ if (VERBOSE) System.out.println("\t" + result);+ }+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);++ numHighlights = 0;+ for (int i = 0; i < hits.totalHits.value; i++) {+ final int docId = hits.scoreDocs[i].doc;+ final Document doc = searcher.doc(docId);+ String text = doc.get(FIELD_NAME);+ TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);+ Highlighter highlighter = getHighlighter(query, FIELD_NAME, TestHighlighter.this);+ highlighter.getBestFragment(tokenStream, text);+ }+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);++ numHighlights = 0;+ for (int i = 0; i < hits.totalHits.value; i++) {+ final int docId = hits.scoreDocs[i].doc;+ final Document doc = searcher.doc(docId);+ String text = doc.get(FIELD_NAME);+ TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);++ Highlighter highlighter = getHighlighter(query, FIELD_NAME, TestHighlighter.this);+ highlighter.getBestFragments(tokenStream, text, 10);+ }+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();-}public void testNotRewriteMultiTermQuery() throws IOException {// field "bar": (not the field we ultimately want to extract)- MultiTermQuery mtq = new TermRangeQuery("bar", new BytesRef("aa"), new BytesRef("zz"), true, true) ;- WeightedSpanTermExtractor extractor = new WeightedSpanTermExtractor() {- @Override- protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms) throws IOException {- assertEquals(mtq, query);- super.extract(query, boost, terms);- }- };+ MultiTermQuery mtq =+ new TermRangeQuery("bar", new BytesRef("aa"), new BytesRef("zz"), true, true);+ WeightedSpanTermExtractor extractor =+ new WeightedSpanTermExtractor() {+ @Override+ protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms)+ throws IOException {+ assertEquals(mtq, query);+ super.extract(query, boost, terms);+ }+ };extractor.setExpandMultiTermQuery(true);extractor.setMaxDocCharsToAnalyze(51200);extractor.getWeightedSpanTerms(- mtq, 3, new CannedTokenStream(new Token("aa",0,2), new Token("bb", 2,4)), "foo"); // field "foo"+ mtq,+ 3,+ new CannedTokenStream(new Token("aa", 0, 2), new Token("bb", 2, 4)),+ "foo"); // field "foo"}public void testGetBestSingleFragmentWithWeights() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- WeightedSpanTerm[] wTerms = new WeightedSpanTerm[2];- wTerms[0] = new WeightedSpanTerm(10f, "hello");-- List<PositionSpan> positionSpans = new ArrayList<>();- positionSpans.add(new PositionSpan(0, 0));- wTerms[0].addPositionSpans(positionSpans);-- wTerms[1] = new WeightedSpanTerm(1f, "kennedy");- positionSpans = new ArrayList<>();- positionSpans.add(new PositionSpan(14, 14));- wTerms[1].addPositionSpans(positionSpans);-- Highlighter highlighter = getHighlighter(wTerms, TestHighlighter.this);// new- // Highlighter(new- // QueryTermScorer(wTerms));- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);- highlighter.setTextFragmenter(new SimpleFragmenter(2));-- String result = highlighter.getBestFragment(tokenStream, texts[0]).trim();- assertTrue("Failed to find best section using weighted terms. Found: [" + result + "]",- "<B>Hello</B>".equals(result));-- // readjust weights- wTerms[1].setWeight(50f);- tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);- highlighter = getHighlighter(wTerms, TestHighlighter.this);- highlighter.setTextFragmenter(new SimpleFragmenter(2));-- result = highlighter.getBestFragment(tokenStream, texts[0]).trim();- assertTrue("Failed to find best section using weighted terms. Found: " + result,- "<B>kennedy</B>".equals(result));- }-- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ WeightedSpanTerm[] wTerms = new WeightedSpanTerm[2];+ wTerms[0] = new WeightedSpanTerm(10f, "hello");++ List<PositionSpan> positionSpans = new ArrayList<>();+ positionSpans.add(new PositionSpan(0, 0));+ wTerms[0].addPositionSpans(positionSpans);++ wTerms[1] = new WeightedSpanTerm(1f, "kennedy");+ positionSpans = new ArrayList<>();+ positionSpans.add(new PositionSpan(14, 14));+ wTerms[1].addPositionSpans(positionSpans);++ Highlighter highlighter = getHighlighter(wTerms, TestHighlighter.this); // new+ // Highlighter(new+ // QueryTermScorer(wTerms));+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);+ highlighter.setTextFragmenter(new SimpleFragmenter(2));++ String result = highlighter.getBestFragment(tokenStream, texts[0]).trim();+ assertTrue(+ "Failed to find best section using weighted terms. Found: [" + result + "]",+ "<B>Hello</B>".equals(result));++ // readjust weights+ wTerms[1].setWeight(50f);+ tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);+ highlighter = getHighlighter(wTerms, TestHighlighter.this);+ highlighter.setTextFragmenter(new SimpleFragmenter(2));++ result = highlighter.getBestFragment(tokenStream, texts[0]).trim();+ assertTrue(+ "Failed to find best section using weighted terms. Found: " + result,+ "<B>kennedy</B>".equals(result));+ }+ };helper.start();-}// tests a "complex" analyzer that produces multiple// overlapping tokenspublic void testOverlapAnalyzer() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- HashMap<String,String> synonyms = new HashMap<>();- synonyms.put("football", "soccer,footie");- try (Analyzer analyzer = new SynonymAnalyzer(synonyms)) {-- String s = "football-soccer in the euro 2004 footie competition";-- BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add(new TermQuery(new Term("bookid", "football")), Occur.SHOULD);- query.add(new TermQuery(new Term("bookid", "soccer")), Occur.SHOULD);- query.add(new TermQuery(new Term("bookid", "footie")), Occur.SHOULD);-- Highlighter highlighter = getHighlighter(query.build(), null, TestHighlighter.this);-- // Get 3 best fragments and separate with a "..."- TokenStream tokenStream = analyzer.tokenStream(null, s);-- String result = highlighter.getBestFragments(tokenStream, s, 3, "...");- String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";- assertTrue("overlapping analyzer should handle highlights OK, expected:" + expectedResult- + " actual:" + result, expectedResult.equals(result));- }- }-- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ HashMap<String, String> synonyms = new HashMap<>();+ synonyms.put("football", "soccer,footie");+ try (Analyzer analyzer = new SynonymAnalyzer(synonyms)) {++ String s = "football-soccer in the euro 2004 footie competition";++ BooleanQuery.Builder query = new BooleanQuery.Builder();+ query.add(new TermQuery(new Term("bookid", "football")), Occur.SHOULD);+ query.add(new TermQuery(new Term("bookid", "soccer")), Occur.SHOULD);+ query.add(new TermQuery(new Term("bookid", "footie")), Occur.SHOULD);++ Highlighter highlighter = getHighlighter(query.build(), null, TestHighlighter.this);++ // Get 3 best fragments and separate with a "..."+ TokenStream tokenStream = analyzer.tokenStream(null, s);++ String result = highlighter.getBestFragments(tokenStream, s, 3, "...");+ String expectedResult =+ "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";+ assertTrue(+ "overlapping analyzer should handle highlights OK, expected:"+ + expectedResult+ + " actual:"+ + result,+ expectedResult.equals(result));+ }+ }+ };helper.start();-}public void testGetSimpleHighlight() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));- // new Highlighter(TestHighlighter.this, new QueryTermScorer(query));-- for (int i = 0; i < hits.totalHits.value; i++) {- String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);- String result = highlighter.getBestFragment(tokenStream, text);- if (VERBOSE) System.out.println("\t" + result);- }- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 4);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));+ // new Highlighter(TestHighlighter.this, new QueryTermScorer(query));++ for (int i = 0; i < hits.totalHits.value; i++) {+ String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);+ Highlighter highlighter = getHighlighter(query, FIELD_NAME, TestHighlighter.this);+ String result = highlighter.getBestFragment(tokenStream, text);+ if (VERBOSE) System.out.println("\t" + result);+ }+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 4);+ }+ };helper.start();}public void testGetTextFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {-- doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));-- for (int i = 0; i < hits.totalHits.value; i++) {- final int docId = hits.scoreDocs[i].doc;- final Document doc = searcher.doc(docId);- String text = doc.get(FIELD_NAME);- TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);-- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);// new Highlighter(this, new- // QueryTermScorer(query));- highlighter.setTextFragmenter(new SimpleFragmenter(20));- String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);-- tokenStream = analyzer.tokenStream(FIELD_NAME, text);- TextFragment fragmentResults[] = highlighter.getBestTextFragments(tokenStream, text,- true, 10);-- assertTrue("Failed to find correct number of text Fragments: " + fragmentResults.length- + " vs " + stringResults.length, fragmentResults.length == stringResults.length);- for (int j = 0; j < stringResults.length; j++) {- if (VERBOSE) System.out.println(fragmentResults[j]);- assertTrue("Failed to find same text Fragments: " + fragmentResults[j] + " found",- fragmentResults[j].toString().equals(stringResults[j]));-+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {++ doSearching(new TermQuery(new Term(FIELD_NAME, "kennedy")));++ for (int i = 0; i < hits.totalHits.value; i++) {+ final int docId = hits.scoreDocs[i].doc;+ final Document doc = searcher.doc(docId);+ String text = doc.get(FIELD_NAME);+ TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);++ Highlighter highlighter =+ getHighlighter(+ query, FIELD_NAME, TestHighlighter.this); // new Highlighter(this, new+ // QueryTermScorer(query));+ highlighter.setTextFragmenter(new SimpleFragmenter(20));+ String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);++ tokenStream = analyzer.tokenStream(FIELD_NAME, text);+ TextFragment fragmentResults[] =+ highlighter.getBestTextFragments(tokenStream, text, true, 10);++ assertTrue(+ "Failed to find correct number of text Fragments: "+ + fragmentResults.length+ + " vs "+ + stringResults.length,+ fragmentResults.length == stringResults.length);+ for (int j = 0; j < stringResults.length; j++) {+ if (VERBOSE) System.out.println(fragmentResults[j]);+ assertTrue(+ "Failed to find same text Fragments: " + fragmentResults[j] + " found",+ fragmentResults[j].toString().equals(stringResults[j]));+ }+ }}-- }- }- };+ };helper.start();}@@ -1458,122 +1564,142 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte// But we first need to clear the re-used tokenstream components that have enableChecks.analyzer.getReuseStrategy().setReusableComponents(analyzer, FIELD_NAME, null);analyzer.setEnableChecks(false);- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- doSearching(new TermQuery(new Term(FIELD_NAME, "meat")));- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);// new Highlighter(this, new- // QueryTermScorer(query));- highlighter.setMaxDocCharsToAnalyze(30);-- highlighter.getBestFragment(tokenStream, texts[0]);- assertTrue("Setting MaxDocBytesToAnalyze should have prevented "- + "us from finding matches for this record: " + numHighlights + " found",- numHighlights == 0);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ doSearching(new TermQuery(new Term(FIELD_NAME, "meat")));+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);+ Highlighter highlighter =+ getHighlighter(+ query, FIELD_NAME, TestHighlighter.this); // new Highlighter(this, new+ // QueryTermScorer(query));+ highlighter.setMaxDocCharsToAnalyze(30);++ highlighter.getBestFragment(tokenStream, texts[0]);+ assertTrue(+ "Setting MaxDocBytesToAnalyze should have prevented "+ + "us from finding matches for this record: "+ + numHighlights+ + " found",+ numHighlights == 0);+ }+ };helper.start();}public void testMaxSizeHighlightTruncates() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- String goodWord = "goodtoken";- CharacterRunAutomaton stopWords = new CharacterRunAutomaton(Automata.makeString("stoppedtoken"));- // we disable MockTokenizer checks because we will forcefully limit the- // tokenstream and call end() before incrementToken() returns false.- final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);- analyzer.setEnableChecks(false);- TermQuery query = new TermQuery(new Term("data", goodWord));-- String match;- StringBuilder sb = new StringBuilder();- sb.append(goodWord);- for (int i = 0; i < 10000; i++) {- sb.append(" ");- // only one stopword- sb.append("stoppedtoken");- }- SimpleHTMLFormatter fm = new SimpleHTMLFormatter();- Highlighter hg = getHighlighter(query, "data", fm);// new Highlighter(fm,- // new- // QueryTermScorer(query));- hg.setTextFragmenter(new NullFragmenter());- hg.setMaxDocCharsToAnalyze(100);- match = hg.getBestFragment(analyzer, "data", sb.toString());- assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg- .getMaxDocCharsToAnalyze());-- // add another tokenized word to the overrall length - but set way- // beyond- // the length of text under consideration (after a large slug of stop- // words- // + whitespace)- sb.append(" ");- sb.append(goodWord);- match = hg.getBestFragment(analyzer, "data", sb.toString());- assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg- .getMaxDocCharsToAnalyze());- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ String goodWord = "goodtoken";+ CharacterRunAutomaton stopWords =+ new CharacterRunAutomaton(Automata.makeString("stoppedtoken"));+ // we disable MockTokenizer checks because we will forcefully limit the+ // tokenstream and call end() before incrementToken() returns false.+ final MockAnalyzer analyzer =+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);+ analyzer.setEnableChecks(false);+ TermQuery query = new TermQuery(new Term("data", goodWord));++ String match;+ StringBuilder sb = new StringBuilder();+ sb.append(goodWord);+ for (int i = 0; i < 10000; i++) {+ sb.append(" ");+ // only one stopword+ sb.append("stoppedtoken");+ }+ SimpleHTMLFormatter fm = new SimpleHTMLFormatter();+ Highlighter hg = getHighlighter(query, "data", fm); // new Highlighter(fm,+ // new+ // QueryTermScorer(query));+ hg.setTextFragmenter(new NullFragmenter());+ hg.setMaxDocCharsToAnalyze(100);+ match = hg.getBestFragment(analyzer, "data", sb.toString());+ assertTrue(+ "Matched text should be no more than 100 chars in length ",+ match.length() < hg.getMaxDocCharsToAnalyze());++ // add another tokenized word to the overrall length - but set way+ // beyond+ // the length of text under consideration (after a large slug of stop+ // words+ // + whitespace)+ sb.append(" ");+ sb.append(goodWord);+ match = hg.getBestFragment(analyzer, "data", sb.toString());+ assertTrue(+ "Matched text should be no more than 100 chars in length ",+ match.length() < hg.getMaxDocCharsToAnalyze());+ }+ };helper.start();-}-+public void testMaxSizeEndHighlight() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {- @Override- public void run() throws Exception {- CharacterRunAutomaton stopWords = new CharacterRunAutomaton(new RegExp("i[nt]").toAutomaton());- TermQuery query = new TermQuery(new Term("text", "searchterm"));-- String text = "this is a text with searchterm in it";- SimpleHTMLFormatter fm = new SimpleHTMLFormatter();- Highlighter hg = getHighlighter(query, "text", fm);- hg.setTextFragmenter(new NullFragmenter());- hg.setMaxDocCharsToAnalyze(36);- String match = hg.getBestFragment(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords), "text", text);- assertTrue(- "Matched text should contain remainder of text after highlighted query ",- match.endsWith("in it"));- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {+ @Override+ public void run() throws Exception {+ CharacterRunAutomaton stopWords =+ new CharacterRunAutomaton(new RegExp("i[nt]").toAutomaton());+ TermQuery query = new TermQuery(new Term("text", "searchterm"));++ String text = "this is a text with searchterm in it";+ SimpleHTMLFormatter fm = new SimpleHTMLFormatter();+ Highlighter hg = getHighlighter(query, "text", fm);+ hg.setTextFragmenter(new NullFragmenter());+ hg.setMaxDocCharsToAnalyze(36);+ String match =+ hg.getBestFragment(+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords),+ "text",+ text);+ assertTrue(+ "Matched text should contain remainder of text after highlighted query ",+ match.endsWith("in it"));+ }+ };helper.start();}public void testHighlighterWithPhraseQuery() throws IOException, InvalidTokenOffsetsException {final String fieldName = "substring";- final PhraseQuery query = new PhraseQuery(fieldName, new BytesRef[] { new BytesRef("uchu") });+ final PhraseQuery query = new PhraseQuery(fieldName, new BytesRef[] {new BytesRef("uchu")});- assertHighlighting(query, new SimpleHTMLFormatter("<b>", "</b>"), "Buchung", "B<b>uchu</b>ng", fieldName);+ assertHighlighting(+ query, new SimpleHTMLFormatter("<b>", "</b>"), "Buchung", "B<b>uchu</b>ng", fieldName);}- public void testHighlighterWithMultiPhraseQuery() throws IOException, InvalidTokenOffsetsException {+ public void testHighlighterWithMultiPhraseQuery()+ throws IOException, InvalidTokenOffsetsException {final String fieldName = "substring";- final MultiPhraseQuery mpq = new MultiPhraseQuery.Builder()- .add(new Term(fieldName, "uchu")).build();+ final MultiPhraseQuery mpq =+ new MultiPhraseQuery.Builder().add(new Term(fieldName, "uchu")).build();- assertHighlighting(mpq, new SimpleHTMLFormatter("<b>", "</b>"), "Buchung", "B<b>uchu</b>ng", fieldName);+ assertHighlighting(+ mpq, new SimpleHTMLFormatter("<b>", "</b>"), "Buchung", "B<b>uchu</b>ng", fieldName);}- private void assertHighlighting(Query query, Formatter formatter, String text, String expected, String fieldName)+ private void assertHighlighting(+ Query query, Formatter formatter, String text, String expected, String fieldName)throws IOException, InvalidTokenOffsetsException {- final Analyzer analyzer = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- return new TokenStreamComponents(new NGramTokenizer(4, 4));- }- };+ final Analyzer analyzer =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ return new TokenStreamComponents(new NGramTokenizer(4, 4));+ }+ };final QueryScorer fragmentScorer = new QueryScorer(query, fieldName);@@ -1585,118 +1711,133 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}public void testUnRewrittenQuery() throws Exception {- final TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- numHighlights = 0;- // test to show how rewritten query can still be used- searcher = newSearcher(reader);-- BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add(new WildcardQuery(new Term(FIELD_NAME, "jf?")), Occur.SHOULD);- query.add(new WildcardQuery(new Term(FIELD_NAME, "kenned*")), Occur.SHOULD);-- if (VERBOSE) System.out.println("Searching with primitive query");- // forget to set this and...- // query=query.rewrite(reader);- TopDocs hits = searcher.search(query.build(), 1000);-- // create an instance of the highlighter with the tags used to surround- // highlighted text- // QueryHighlightExtractor highlighter = new- // QueryHighlightExtractor(this,- // query, new StandardAnalyzer(TEST_VERSION));-- int maxNumFragmentsRequired = 3;-- for (int i = 0; i < hits.totalHits.value; i++) {- final int docId = hits.scoreDocs[i].doc;- final Document doc = searcher.doc(docId);- String text = doc.get(FIELD_NAME);- TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);- Highlighter highlighter = getHighlighter(query.build(), FIELD_NAME, TestHighlighter.this, false);-- highlighter.setTextFragmenter(new SimpleFragmenter(40));-- String highlightedText = highlighter.getBestFragments(tokenStream, text,- maxNumFragmentsRequired, "...");-- if (VERBOSE) System.out.println(highlightedText);- }- // We expect to have zero highlights if the query is multi-terms and is- // not- // rewritten!- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",- numHighlights == 0);- }- };+ final TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ numHighlights = 0;+ // test to show how rewritten query can still be used+ searcher = newSearcher(reader);++ BooleanQuery.Builder query = new BooleanQuery.Builder();+ query.add(new WildcardQuery(new Term(FIELD_NAME, "jf?")), Occur.SHOULD);+ query.add(new WildcardQuery(new Term(FIELD_NAME, "kenned*")), Occur.SHOULD);++ if (VERBOSE) System.out.println("Searching with primitive query");+ // forget to set this and...+ // query=query.rewrite(reader);+ TopDocs hits = searcher.search(query.build(), 1000);++ // create an instance of the highlighter with the tags used to surround+ // highlighted text+ // QueryHighlightExtractor highlighter = new+ // QueryHighlightExtractor(this,+ // query, new StandardAnalyzer(TEST_VERSION));++ int maxNumFragmentsRequired = 3;++ for (int i = 0; i < hits.totalHits.value; i++) {+ final int docId = hits.scoreDocs[i].doc;+ final Document doc = searcher.doc(docId);+ String text = doc.get(FIELD_NAME);+ TokenStream tokenStream = getAnyTokenStream(FIELD_NAME, docId);+ Highlighter highlighter =+ getHighlighter(query.build(), FIELD_NAME, TestHighlighter.this, false);++ highlighter.setTextFragmenter(new SimpleFragmenter(40));++ String highlightedText =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");++ if (VERBOSE) System.out.println(highlightedText);+ }+ // We expect to have zero highlights if the query is multi-terms and is+ // not+ // rewritten!+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",+ numHighlights == 0);+ }+ };helper.start();}public void testNoFragments() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- doSearching(new TermQuery(new Term(FIELD_NAME, "aninvalidquerywhichshouldyieldnoresults")));-- for (String text : texts) {- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);- Highlighter highlighter = getHighlighter(query, FIELD_NAME,- TestHighlighter.this);- String result = highlighter.getBestFragment(tokenStream, text);- assertNull("The highlight result should be null for text with no query terms", result);- }- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ doSearching(+ new TermQuery(new Term(FIELD_NAME, "aninvalidquerywhichshouldyieldnoresults")));++ for (String text : texts) {+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);+ Highlighter highlighter = getHighlighter(query, FIELD_NAME, TestHighlighter.this);+ String result = highlighter.getBestFragment(tokenStream, text);+ assertNull(+ "The highlight result should be null for text with no query terms", result);+ }+ }+ };helper.start();}- /**- * Demonstrates creation of an XHTML compliant doc using new encoding facilities.- */+ /** Demonstrates creation of an XHTML compliant doc using new encoding facilities. */public void testEncoding() throws Exception {String rawDocContent = "\"Smith & sons' prices < 3 and >4\" claims article";// run the highlighter on the raw content (scorer does not score any tokens// for// highlighting but scores a single fragment for selection- Highlighter highlighter = new Highlighter(this, new SimpleHTMLEncoder(), new Scorer() {- @Override- public void startFragment(TextFragment newFragment) {- }-- @Override- public float getTokenScore() {- return 0;- }-- @Override- public float getFragmentScore() {- return 1;- }-- @Override- public TokenStream init(TokenStream tokenStream) {- return null;- }- });+ Highlighter highlighter =+ new Highlighter(+ this,+ new SimpleHTMLEncoder(),+ new Scorer() {+ @Override+ public void startFragment(TextFragment newFragment) {}++ @Override+ public float getTokenScore() {+ return 0;+ }++ @Override+ public float getFragmentScore() {+ return 1;+ }++ @Override+ public TokenStream init(TokenStream tokenStream) {+ return null;+ }+ });highlighter.setTextFragmenter(new SimpleFragmenter(2000));TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, rawDocContent);String encodedSnippet = highlighter.getBestFragments(tokenStream, rawDocContent, 1, "");// An ugly bit of XML creation:- String xhtml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"- + "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n"- + "<head>\n" + "<title>My Test HTML Document</title>\n" + "</head>\n" + "<body>\n" + "<h2>"- + encodedSnippet + "</h2>\n" + "</body>\n" + "</html>";+ String xhtml =+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"+ + "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n"+ + "<head>\n"+ + "<title>My Test HTML Document</title>\n"+ + "</head>\n"+ + "<body>\n"+ + "<h2>"+ + encodedSnippet+ + "</h2>\n"+ + "</body>\n"+ + "</html>";// now an ugly built of XML parsing to test the snippet is encoded OKDocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();DocumentBuilder db = dbf.newDocumentBuilder();- org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8)));+ org.w3c.dom.Document doc =+ db.parse(new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8)));Element root = doc.getDocumentElement();NodeList nodes = root.getElementsByTagName("body");Element body = (Element) nodes.item(0);@@ -1707,50 +1848,51 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}public void testFieldSpecificHighlighting() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- String docMainText = "fred is one of the people";-- BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add(new TermQuery(new Term(FIELD_NAME, "fred")), Occur.SHOULD);- query.add(new TermQuery(new Term("category", "people")), Occur.SHOULD);-- // highlighting respects fieldnames used in query-- Scorer fieldSpecificScorer = null;- if (mode == TestHighlightRunner.QUERY) {- fieldSpecificScorer = new QueryScorer(query.build(), FIELD_NAME);- } else if (mode == TestHighlightRunner.QUERY_TERM) {- fieldSpecificScorer = new QueryTermScorer(query.build(), "contents");- }- Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),- fieldSpecificScorer);- fieldSpecificHighlighter.setTextFragmenter(new NullFragmenter());- String result = fieldSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);- assertEquals("Should match", result, "<B>fred</B> is one of the people");-- // highlighting does not respect fieldnames used in query- Scorer fieldInSpecificScorer = null;- if (mode == TestHighlightRunner.QUERY) {- fieldInSpecificScorer = new QueryScorer(query.build(), null);- } else if (mode == TestHighlightRunner.QUERY_TERM) {- fieldInSpecificScorer = new QueryTermScorer(query.build());- }-- Highlighter fieldInSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),- fieldInSpecificScorer);- fieldInSpecificHighlighter.setTextFragmenter(new NullFragmenter());- result = fieldInSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);- assertEquals("Should match", result, "<B>fred</B> is one of the <B>people</B>");-- reader.close();- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ String docMainText = "fred is one of the people";++ BooleanQuery.Builder query = new BooleanQuery.Builder();+ query.add(new TermQuery(new Term(FIELD_NAME, "fred")), Occur.SHOULD);+ query.add(new TermQuery(new Term("category", "people")), Occur.SHOULD);++ // highlighting respects fieldnames used in query++ Scorer fieldSpecificScorer = null;+ if (mode == TestHighlightRunner.QUERY) {+ fieldSpecificScorer = new QueryScorer(query.build(), FIELD_NAME);+ } else if (mode == TestHighlightRunner.QUERY_TERM) {+ fieldSpecificScorer = new QueryTermScorer(query.build(), "contents");+ }+ Highlighter fieldSpecificHighlighter =+ new Highlighter(new SimpleHTMLFormatter(), fieldSpecificScorer);+ fieldSpecificHighlighter.setTextFragmenter(new NullFragmenter());+ String result =+ fieldSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);+ assertEquals("Should match", result, "<B>fred</B> is one of the people");++ // highlighting does not respect fieldnames used in query+ Scorer fieldInSpecificScorer = null;+ if (mode == TestHighlightRunner.QUERY) {+ fieldInSpecificScorer = new QueryScorer(query.build(), null);+ } else if (mode == TestHighlightRunner.QUERY_TERM) {+ fieldInSpecificScorer = new QueryTermScorer(query.build());+ }++ Highlighter fieldInSpecificHighlighter =+ new Highlighter(new SimpleHTMLFormatter(), fieldInSpecificScorer);+ fieldInSpecificHighlighter.setTextFragmenter(new NullFragmenter());+ result = fieldInSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);+ assertEquals("Should match", result, "<B>fred</B> is one of the <B>people</B>");++ reader.close();+ }+ };helper.start();-}protected TokenStream getTS2() {@@ -1759,8 +1901,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteIterator<Token> iter;List<Token> lst;private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute posIncrAtt =+ addAttribute(PositionIncrementAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);+{lst = new ArrayList<>();Token t;@@ -1784,7 +1928,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte@Overridepublic boolean incrementToken() {- if(iter.hasNext()) {+ if (iter.hasNext()) {Token token = iter.next();clearAttributes();termAtt.setEmpty().append(token);@@ -1810,8 +1954,10 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteIterator<Token> iter;List<Token> lst;private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);- private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute posIncrAtt =+ addAttribute(PositionIncrementAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);+{lst = new ArrayList<>();Token t;@@ -1835,7 +1981,7 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte@Overridepublic boolean incrementToken() {- if(iter.hasNext()) {+ if (iter.hasNext()) {Token token = iter.next();clearAttributes();termAtt.setEmpty().append(token);@@ -1855,139 +2001,151 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte}public void testOverlapAnalyzer2() throws Exception {- TestHighlightRunner helper = new TestHighlightRunner() {-- @Override- public void run() throws Exception {- String s = "Hi-Speed10 foo";-- Query query;- Highlighter highlighter;- String result;-- query = new TermQuery(new Term("text", "foo"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("Hi-Speed10 <B>foo</B>", result);-- query = new TermQuery(new Term("text", "10"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("Hi-Speed<B>10</B> foo", result);-- query = new TermQuery(new Term("text", "hi"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("<B>Hi</B>-Speed10 foo", result);-- query = new TermQuery(new Term("text", "speed"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("Hi-<B>Speed</B>10 foo", result);-- query = new TermQuery(new Term("text", "hispeed"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("<B>Hi-Speed</B>10 foo", result);-- BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();- booleanQuery.add(new TermQuery(new Term("text", "hi")), Occur.SHOULD);- booleanQuery.add(new TermQuery(new Term("text", "speed")), Occur.SHOULD);-- query = booleanQuery.build();- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2(), s, 3, "...");- assertEquals("<B>Hi-Speed</B>10 foo", result);-- // ///////////////// same tests, just put the bigger overlapping token- // first- query = new TermQuery(new Term("text", "foo"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("Hi-Speed10 <B>foo</B>", result);-- query = new TermQuery(new Term("text", "10"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("Hi-Speed<B>10</B> foo", result);-- query = new TermQuery(new Term("text", "hi"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("<B>Hi</B>-Speed10 foo", result);-- query = new TermQuery(new Term("text", "speed"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("Hi-<B>Speed</B>10 foo", result);-- query = new TermQuery(new Term("text", "hispeed"));- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("<B>Hi-Speed</B>10 foo", result);-- query = booleanQuery.build();- highlighter = getHighlighter(query, "text", TestHighlighter.this);- result = highlighter.getBestFragments(getTS2a(), s, 3, "...");- assertEquals("<B>Hi-Speed</B>10 foo", result);- }- };+ TestHighlightRunner helper =+ new TestHighlightRunner() {++ @Override+ public void run() throws Exception {+ String s = "Hi-Speed10 foo";++ Query query;+ Highlighter highlighter;+ String result;++ query = new TermQuery(new Term("text", "foo"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("Hi-Speed10 <B>foo</B>", result);++ query = new TermQuery(new Term("text", "10"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("Hi-Speed<B>10</B> foo", result);++ query = new TermQuery(new Term("text", "hi"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("<B>Hi</B>-Speed10 foo", result);++ query = new TermQuery(new Term("text", "speed"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("Hi-<B>Speed</B>10 foo", result);++ query = new TermQuery(new Term("text", "hispeed"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("<B>Hi-Speed</B>10 foo", result);++ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();+ booleanQuery.add(new TermQuery(new Term("text", "hi")), Occur.SHOULD);+ booleanQuery.add(new TermQuery(new Term("text", "speed")), Occur.SHOULD);++ query = booleanQuery.build();+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2(), s, 3, "...");+ assertEquals("<B>Hi-Speed</B>10 foo", result);++ // ///////////////// same tests, just put the bigger overlapping token+ // first+ query = new TermQuery(new Term("text", "foo"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("Hi-Speed10 <B>foo</B>", result);++ query = new TermQuery(new Term("text", "10"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("Hi-Speed<B>10</B> foo", result);++ query = new TermQuery(new Term("text", "hi"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("<B>Hi</B>-Speed10 foo", result);++ query = new TermQuery(new Term("text", "speed"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("Hi-<B>Speed</B>10 foo", result);++ query = new TermQuery(new Term("text", "hispeed"));+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("<B>Hi-Speed</B>10 foo", result);++ query = booleanQuery.build();+ highlighter = getHighlighter(query, "text", TestHighlighter.this);+ result = highlighter.getBestFragments(getTS2a(), s, 3, "...");+ assertEquals("<B>Hi-Speed</B>10 foo", result);+ }+ };helper.start();}-+private Directory dir2;private Analyzer a;-+public void testWeightedTermsWithDeletes() throws IOException, InvalidTokenOffsetsException {makeIndex();deleteDocument();searchIndex();}-+private void makeIndex() throws IOException {- IndexWriter writer = new IndexWriter(dir1, new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));- writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );- writer.addDocument( doc( "t_text1", "more random words for second field del" ) );- writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );- writer.addDocument( doc( "t_text1", "more random words for second field" ) );+ IndexWriter writer =+ new IndexWriter(+ dir1,+ new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ writer.addDocument(doc("t_text1", "random words for highlighting tests del"));+ writer.addDocument(doc("t_text1", "more random words for second field del"));+ writer.addDocument(doc("t_text1", "random words for highlighting tests del"));+ writer.addDocument(doc("t_text1", "more random words for second field"));writer.forceMerge(1);writer.close();}-+private void deleteDocument() throws IOException {- IndexWriter writer = new IndexWriter(dir1, new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));- writer.deleteDocuments( new Term( "t_text1", "del" ) );+ IndexWriter writer =+ new IndexWriter(+ dir1,+ new IndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))+ .setOpenMode(OpenMode.APPEND));+ writer.deleteDocuments(new Term("t_text1", "del"));// To see negative idf, keep comment the following line- //writer.forceMerge(1);+ // writer.forceMerge(1);writer.close();}-+private void searchIndex() throws IOException, InvalidTokenOffsetsException {Query query = new TermQuery(new Term("t_text1", "random"));IndexReader reader = DirectoryReader.open(dir1);IndexSearcher searcher = newSearcher(reader);// This scorer can return negative idf -> null fragment- Scorer scorer = new QueryTermScorer( query, searcher.getIndexReader(), "t_text1" );+ Scorer scorer = new QueryTermScorer(query, searcher.getIndexReader(), "t_text1");// This scorer doesn't use idf (patch version)- //Scorer scorer = new QueryTermScorer( query, "t_text1" );- Highlighter h = new Highlighter( scorer );+ // Scorer scorer = new QueryTermScorer( query, "t_text1" );+ Highlighter h = new Highlighter(scorer);TopDocs hits = searcher.search(query, 10);- for( int i = 0; i < hits.totalHits.value; i++ ){- Document doc = searcher.doc( hits.scoreDocs[i].doc );- String result = h.getBestFragment( a, "t_text1", doc.get( "t_text1" ));- if (VERBOSE) System.out.println("result:" + result);+ for (int i = 0; i < hits.totalHits.value; i++) {+ Document doc = searcher.doc(hits.scoreDocs[i].doc);+ String result = h.getBestFragment(a, "t_text1", doc.get("t_text1"));+ if (VERBOSE) System.out.println("result:" + result);assertEquals("more <B>random</B> words for second field", result);}reader.close();}- /** We can highlight based on payloads. It's supported both via term vectors and MemoryIndex since Lucene 5. */+ /**+ * We can highlight based on payloads. It's supported both via term vectors and MemoryIndex since+ * Lucene 5.+ */public void testPayloadQuery() throws IOException, InvalidTokenOffsetsException {- final String text = "random words and words";//"words" at positions 1 & 4+ final String text = "random words and words"; // "words" at positions 1 & 4- Analyzer analyzer = new MockPayloadAnalyzer();//sets payload to "pos: X" (where X is position #)+ Analyzer analyzer =+ new MockPayloadAnalyzer(); // sets payload to "pos: X" (where X is position #)try (IndexWriter writer = new IndexWriter(dir1, new IndexWriterConfig(analyzer))) {writer.deleteAll();Document doc = new Document();@@ -1997,8 +2155,11 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattewriter.commit();}try (IndexReader reader = DirectoryReader.open(dir1)) {- Query query = new SpanPayloadCheckQuery(new SpanTermQuery(new Term(FIELD_NAME, "words")),- Collections.singletonList(new BytesRef("pos: 1")));//just match the first "word" occurrence+ Query query =+ new SpanPayloadCheckQuery(+ new SpanTermQuery(new Term(FIELD_NAME, "words")),+ Collections.singletonList(+ new BytesRef("pos: 1"))); // just match the first "word" occurrenceIndexSearcher searcher = newSearcher(reader);QueryScorer scorer = new QueryScorer(query, searcher.getIndexReader(), FIELD_NAME);scorer.setUsePayloads(true);@@ -2006,12 +2167,13 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteTopDocs hits = searcher.search(query, 10);assertEquals(1, hits.scoreDocs.length);- TokenStream stream = TokenSources.getAnyTokenStream(searcher.getIndexReader(), 0, FIELD_NAME, analyzer);+ TokenStream stream =+ TokenSources.getAnyTokenStream(searcher.getIndexReader(), 0, FIELD_NAME, analyzer);if (random().nextBoolean()) {- stream = new CachingTokenFilter(stream);//conceals detection of TokenStreamFromTermVector+ stream = new CachingTokenFilter(stream); // conceals detection of TokenStreamFromTermVector}String result = h.getBestFragment(stream, text);- assertEquals("random <B>words</B> and words", result);//only highlight first "word"+ assertEquals("random <B>words</B> and words", result); // only highlight first "word"}}@@ -2033,8 +2195,8 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehits = searcher.search(query, 1000);}- public void assertExpectedHighlightCount(final int maxNumFragmentsRequired,- final int expectedHighlights) throws Exception {+ public void assertExpectedHighlightCount(+ final int maxNumFragmentsRequired, final int expectedHighlights) throws Exception {for (int i = 0; i < hits.totalHits.value; i++) {String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);@@ -2043,11 +2205,12 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattehighlighter.setTextFragmenter(new SimpleFragmenter(40));- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- "...");+ String result =+ highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");if (VERBOSE) System.out.println("\t" + result);- assertTrue("Failed to find correct number of highlights " + numHighlights + " found",+ assertTrue(+ "Failed to find correct number of highlights " + numHighlights + " found",numHighlights == expectedHighlights);}}@@ -2056,15 +2219,17 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formattepublic void setUp() throws Exception {super.setUp();- //Not many use this setup:+ // Not many use this setup:a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);dir1 = newDirectory();- //Most tests use this setup:- analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);+ // Most tests use this setup:+ analyzer =+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);dir2 = newDirectory();fieldType = random().nextBoolean() ? FIELD_TYPE_TV : TextField.TYPE_STORED;- IndexWriter writer = new IndexWriter(dir2, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()));+ IndexWriter writer =+ new IndexWriter(dir2, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()));for (String text : texts) {writer.addDocument(doc(FIELD_NAME, text));@@ -2094,12 +2259,12 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements FormatteDocument childDoc = doc(FIELD_NAME, "child document");Document parentDoc = doc(FIELD_NAME, "parent document");writer.addDocuments(Arrays.asList(childDoc, parentDoc));-+writer.forceMerge(1);writer.close();reader = DirectoryReader.open(dir2);- //Misc:+ // Misc:numHighlights = 0;}@@ -2113,15 +2278,15 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatteprivate Document doc(String name, String value) {Document d = new Document();- d.add(new Field(name, value, fieldType));//fieldType is randomly chosen for term vectors in setUp+ d.add(+ new Field(+ name, value, fieldType)); // fieldType is randomly chosen for term vectors in setUpreturn d;}- private static Token createToken(String term, int start, int offset)- {+ private static Token createToken(String term, int start, int offset) {return new Token(term, start, offset);}-}// ===================================================================@@ -2135,15 +2300,15 @@ public class TestHighlighter extends BaseTokenStreamTestCase implements Formatte// ===================================================================final class SynonymAnalyzer extends Analyzer {- private Map<String,String> synonyms;+ private Map<String, String> synonyms;- public SynonymAnalyzer(Map<String,String> synonyms) {+ public SynonymAnalyzer(Map<String, String> synonyms) {this.synonyms = synonyms;}/** (non-Javadoc)- *+ ** @see org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String,* java.io.Reader)*/@@ -2159,7 +2324,6 @@ final class SynonymAnalyzer extends Analyzer {/*** Expands a token stream with synonyms (TODO - make the synonyms analyzed by choice of analyzer)- **/final class SynonymTokenizer extends TokenStream {private final TokenStream realStream;@@ -2193,13 +2357,13 @@ final class SynonymTokenizer extends TokenStream {if (!next) {return false;}- //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());+ // Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());clearAttributes();termAtt.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());- String expansions = synonyms.get(realTermAtt.toString());+ String expansions = synonyms.get(realTermAtt.toString());if (expansions == null) {return true;}@@ -2209,7 +2373,7 @@ final class SynonymTokenizer extends TokenStream {currentRealToken.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());}-+return true;} else {String tok = st.nextToken();@@ -2223,7 +2387,6 @@ final class SynonymTokenizer extends TokenStream {}return true;}-}@Override@@ -2246,36 +2409,37 @@ final class SynonymTokenizer extends TokenStream {this.realStream.close();}- static abstract class TestHighlightRunner {+ abstract static class TestHighlightRunner {static final int QUERY = 0;static final int QUERY_TERM = 1;int mode = QUERY;Fragmenter frag = new SimpleFragmenter(20);-+public Highlighter getHighlighter(Query query, String fieldName, Formatter formatter) {return getHighlighter(query, fieldName, formatter, true);}-- public Highlighter getHighlighter(Query query, String fieldName, Formatter formatter, boolean expanMultiTerm) {++ public Highlighter getHighlighter(+ Query query, String fieldName, Formatter formatter, boolean expanMultiTerm) {Scorer scorer;if (mode == QUERY) {scorer = new QueryScorer(query, fieldName);- if(!expanMultiTerm) {- ((QueryScorer)scorer).setExpandMultiTermQuery(false);+ if (!expanMultiTerm) {+ ((QueryScorer) scorer).setExpandMultiTermQuery(false);}} else if (mode == QUERY_TERM) {scorer = new QueryTermScorer(query);} else {throw new RuntimeException("Unknown highlight mode");}-+return new Highlighter(formatter, scorer);}Highlighter getHighlighter(WeightedTerm[] weightedTerms, Formatter formatter) {if (mode == QUERY) {- return new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));+ return new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));} else if (mode == QUERY_TERM) {return new Highlighter(formatter, new QueryTermScorer(weightedTerms));@@ -2284,12 +2448,19 @@ final class SynonymTokenizer extends TokenStream {}}- void doStandardHighlights(Analyzer analyzer, IndexSearcher searcher, TopDocs hits, Query query, Formatter formatter)- throws Exception {+ void doStandardHighlights(+ Analyzer analyzer, IndexSearcher searcher, TopDocs hits, Query query, Formatter formatter)+ throws Exception {doStandardHighlights(analyzer, searcher, hits, query, formatter, false);}-- void doStandardHighlights(Analyzer analyzer, IndexSearcher searcher, TopDocs hits, Query query, Formatter formatter, boolean expandMT)++ void doStandardHighlights(+ Analyzer analyzer,+ IndexSearcher searcher,+ TopDocs hits,+ Query query,+ Formatter formatter,+ boolean expandMT)throws Exception {for (int i = 0; i < hits.totalHits.value; i++) {@@ -2300,8 +2471,12 @@ final class SynonymTokenizer extends TokenStream {String fragmentSeparator = "...";Scorer scorer = null;TokenStream tokenStream =- TokenSources.getTokenStream(TestHighlighter.FIELD_NAME,- searcher.getIndexReader().getTermVectors(docId), text, analyzer, -1);+ TokenSources.getTokenStream(+ TestHighlighter.FIELD_NAME,+ searcher.getIndexReader().getTermVectors(docId),+ text,+ analyzer,+ -1);if (mode == QUERY) {scorer = new QueryScorer(query);} else if (mode == QUERY_TERM) {@@ -2309,8 +2484,9 @@ final class SynonymTokenizer extends TokenStream {}Highlighter highlighter = new Highlighter(formatter, scorer);highlighter.setTextFragmenter(frag);- String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,- fragmentSeparator);+ String result =+ highlighter.getBestFragments(+ tokenStream, text, maxNumFragmentsRequired, fragmentSeparator);if (LuceneTestCase.VERBOSE) System.out.println("\t" + result);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.javaindex 8e92456c1dd..1c52e02d2a3 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestHighlighterPhrase.java@@ -17,7 +17,6 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenFilter;import org.apache.lucene.analysis.MockTokenizer;@@ -56,8 +55,10 @@ public class TestHighlighterPhrase extends LuceneTestCase {public void testConcurrentPhrase() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox jumped";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ final IndexWriter indexWriter =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -76,14 +77,15 @@ public class TestHighlighterPhrase extends LuceneTestCase {final PhraseQuery phraseQuery = new PhraseQuery(FIELD, "fox", "jumped");TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals(highlighter.getBestFragment(new TokenStreamConcurrent(),- TEXT), highlighter.getBestFragment(tokenStream, TEXT));+ assertEquals(+ highlighter.getBestFragment(new TokenStreamConcurrent(), TEXT),+ highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();@@ -93,8 +95,10 @@ public class TestHighlighterPhrase extends LuceneTestCase {public void testConcurrentSpan() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox jumped";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ final IndexWriter indexWriter =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));try {final Document document = new Document();@@ -111,45 +115,55 @@ public class TestHighlighterPhrase extends LuceneTestCase {try {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);- final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD, "fox")),- new SpanTermQuery(new Term(FIELD, "jumped")) }, 0, true);+ final Query phraseQuery =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD, "fox")),+ new SpanTermQuery(new Term(FIELD, "jumped"))+ },+ 0,+ true);final FixedBitSet bitset = new FixedBitSet(indexReader.maxDoc());- indexSearcher.search(phraseQuery, new SimpleCollector() {- private int baseDoc;-- @Override- public void collect(int i) {- bitset.set(this.baseDoc + i);- }-- @Override- protected void doSetNextReader(LeafReaderContext context) throws IOException {- this.baseDoc = context.docBase;- }-- @Override- public void setScorer(Scorable scorer) {- // Do Nothing- }-- @Override- public ScoreMode scoreMode() {- return ScoreMode.COMPLETE_NO_SCORES;- }- });+ indexSearcher.search(+ phraseQuery,+ new SimpleCollector() {+ private int baseDoc;++ @Override+ public void collect(int i) {+ bitset.set(this.baseDoc + i);+ }++ @Override+ protected void doSetNextReader(LeafReaderContext context) throws IOException {+ this.baseDoc = context.docBase;+ }++ @Override+ public void setScorer(Scorable scorer) {+ // Do Nothing+ }++ @Override+ public ScoreMode scoreMode() {+ return ScoreMode.COMPLETE_NO_SCORES;+ }+ });assertEquals(1, bitset.cardinality());final int maxDoc = indexReader.maxDoc();- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));- for (int position = bitset.nextSetBit(0); position < maxDoc-1; position = bitset- .nextSetBit(position + 1)) {+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));+ for (int position = bitset.nextSetBit(0);+ position < maxDoc - 1;+ position = bitset.nextSetBit(position + 1)) {assertEquals(0, position);final TokenStream tokenStream =- TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(position), -1);- assertEquals(highlighter.getBestFragment(new TokenStreamConcurrent(),- TEXT), highlighter.getBestFragment(tokenStream, TEXT));+ TokenSources.getTermVectorTokenStreamOrNull(+ FIELD, indexReader.getTermVectors(position), -1);+ assertEquals(+ highlighter.getBestFragment(new TokenStreamConcurrent(), TEXT),+ highlighter.getBestFragment(tokenStream, TEXT));}} finally {indexReader.close();@@ -160,8 +174,10 @@ public class TestHighlighterPhrase extends LuceneTestCase {public void testSparsePhrase() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ final IndexWriter indexWriter =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));try {final Document document = new Document();@@ -181,9 +197,9 @@ public class TestHighlighterPhrase extends LuceneTestCase {final PhraseQuery phraseQuery = new PhraseQuery(FIELD, "did", "jump");TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(0, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);assertEquals(@@ -198,8 +214,10 @@ public class TestHighlighterPhrase extends LuceneTestCase {public void testSparsePhraseWithNoPositions() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ final IndexWriter indexWriter =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));try {final Document document = new Document();@@ -218,13 +236,13 @@ public class TestHighlighterPhrase extends LuceneTestCase {final PhraseQuery phraseQuery = new PhraseQuery(1, FIELD, "did", "jump");TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals("the fox <B>did</B> not <B>jump</B>", highlighter- .getBestFragment(tokenStream, TEXT));+ assertEquals(+ "the fox <B>did</B> not <B>jump</B>", highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();@@ -234,8 +252,10 @@ public class TestHighlighterPhrase extends LuceneTestCase {public void testSparseSpan() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));+ final IndexWriter indexWriter =+ new IndexWriter(+ directory,+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -251,15 +271,20 @@ public class TestHighlighterPhrase extends LuceneTestCase {try {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);- final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD, "did")),- new SpanTermQuery(new Term(FIELD, "jump")) }, 0, true);+ final Query phraseQuery =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD, "did")),+ new SpanTermQuery(new Term(FIELD, "jump"))+ },+ 0,+ true);TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(0, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);assertEquals(@@ -271,14 +296,13 @@ public class TestHighlighterPhrase extends LuceneTestCase {}}- //shows the need to sum the increments in WeightedSpanTermExtractor+ // shows the need to sum the increments in WeightedSpanTermExtractorpublic void testStopWords() throws IOException, InvalidTokenOffsetsException {- MockAnalyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true,- MockTokenFilter.ENGLISH_STOPSET);+ MockAnalyzer stopAnalyzer =+ new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);final String TEXT = "the ab the the cd the the the ef the";final Directory directory = newDirectory();- try (IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(stopAnalyzer))) {+ try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(stopAnalyzer))) {final Document document = new Document();document.add(newTextField(FIELD, TEXT, Store.YES));indexWriter.addDocument(document);@@ -286,32 +310,33 @@ public class TestHighlighterPhrase extends LuceneTestCase {try (IndexReader indexReader = DirectoryReader.open(directory)) {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);- //equivalent of "ab the the cd the the the ef"- final PhraseQuery phraseQuery = new PhraseQuery.Builder()- .add(new Term(FIELD, "ab"), 0)- .add(new Term(FIELD, "cd"), 3)- .add(new Term(FIELD, "ef"), 7).build();+ // equivalent of "ab the the cd the the the ef"+ final PhraseQuery phraseQuery =+ new PhraseQuery.Builder()+ .add(new Term(FIELD, "ab"), 0)+ .add(new Term(FIELD, "cd"), 3)+ .add(new Term(FIELD, "ef"), 7)+ .build();TopDocs hits = indexSearcher.search(phraseQuery, 100);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));assertEquals(1, highlighter.getBestFragments(stopAnalyzer, FIELD, TEXT, 10).length);} finally {directory.close();}}-- //shows the need to require inOrder if getSlop() == 0, not if final slop == 0- //in WeightedSpanTermExtractor++ // shows the need to require inOrder if getSlop() == 0, not if final slop == 0+ // in WeightedSpanTermExtractorpublic void testInOrderWithStopWords() throws IOException, InvalidTokenOffsetsException {- MockAnalyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true,- MockTokenFilter.ENGLISH_STOPSET);+ MockAnalyzer stopAnalyzer =+ new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);final String TEXT = "the cd the ab the the the the the the the ab the cd the";final Directory directory = newDirectory();- try (IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(stopAnalyzer))) {+ try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(stopAnalyzer))) {final Document document = new Document();document.add(newTextField(FIELD, TEXT, Store.YES));indexWriter.addDocument(document);@@ -319,22 +344,25 @@ public class TestHighlighterPhrase extends LuceneTestCase {try (IndexReader indexReader = DirectoryReader.open(directory)) {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);- //equivalent of "ab the cd"- final PhraseQuery phraseQuery = new PhraseQuery.Builder()- .add(new Term(FIELD, "ab"), 0)- .add(new Term(FIELD, "cd"), 2).build();+ // equivalent of "ab the cd"+ final PhraseQuery phraseQuery =+ new PhraseQuery.Builder()+ .add(new Term(FIELD, "ab"), 0)+ .add(new Term(FIELD, "cd"), 2)+ .build();TopDocs hits = indexSearcher.search(phraseQuery, 100);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));String[] frags = highlighter.getBestFragments(stopAnalyzer, FIELD, TEXT, 10);assertEquals(1, frags.length);- assertTrue("contains <B>ab</B> the <B>cd</B>",- (frags[0].contains("<B>ab</B> the <B>cd</B>")));- assertTrue("does not contain <B>cd</B> the <B>ab</B>",+ assertTrue(+ "contains <B>ab</B> the <B>cd</B>", (frags[0].contains("<B>ab</B> the <B>cd</B>")));+ assertTrue(+ "does not contain <B>cd</B> the <B>ab</B>",(!frags[0].contains("<B>cd</B> the <B>ab</B>")));} finally {directory.close();@@ -348,7 +376,8 @@ public class TestHighlighterPhrase extends LuceneTestCase {private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);- private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute positionIncrementAttribute =+ addAttribute(PositionIncrementAttribute.class);public TokenStreamSparse() {reset();@@ -362,21 +391,21 @@ public class TestHighlighterPhrase extends LuceneTestCase {}clearAttributes();termAttribute.setEmpty().append(this.tokens[i]);- offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]- .endOffset());- positionIncrementAttribute.setPositionIncrement(this.tokens[i]- .getPositionIncrement());+ offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i].endOffset());+ positionIncrementAttribute.setPositionIncrement(this.tokens[i].getPositionIncrement());return true;}@Overridepublic void reset() {this.i = -1;- this.tokens = new Token[] {- new Token("the", 0, 3),- new Token("fox", 4, 7),- new Token("did", 8, 11),- new Token("jump", 16, 20) };+ this.tokens =+ new Token[] {+ new Token("the", 0, 3),+ new Token("fox", 4, 7),+ new Token("did", 8, 11),+ new Token("jump", 16, 20)+ };this.tokens[3].setPositionIncrement(2);}}@@ -388,7 +417,8 @@ public class TestHighlighterPhrase extends LuceneTestCase {private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);- private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute positionIncrementAttribute =+ addAttribute(PositionIncrementAttribute.class);public TokenStreamConcurrent() {reset();@@ -402,23 +432,22 @@ public class TestHighlighterPhrase extends LuceneTestCase {}clearAttributes();termAttribute.setEmpty().append(this.tokens[i]);- offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]- .endOffset());- positionIncrementAttribute.setPositionIncrement(this.tokens[i]- .getPositionIncrement());+ offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i].endOffset());+ positionIncrementAttribute.setPositionIncrement(this.tokens[i].getPositionIncrement());return true;}@Overridepublic void reset() {this.i = -1;- this.tokens = new Token[] {- new Token("the", 0, 3),- new Token("fox", 4, 7),- new Token("jump", 8, 14),- new Token("jumped", 8, 14) };+ this.tokens =+ new Token[] {+ new Token("the", 0, 3),+ new Token("fox", 4, 7),+ new Token("jump", 8, 14),+ new Token("jumped", 8, 14)+ };this.tokens[3].setPositionIncrement(0);}}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestMisses.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestMisses.javaindex 02cdc04c149..e68bab6d9df 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestMisses.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestMisses.java@@ -17,7 +17,6 @@package org.apache.lucene.search.highlight;import java.io.IOException;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -36,8 +35,10 @@ public class TestMisses extends LuceneTestCase {public void testTermQuery() throws IOException, InvalidTokenOffsetsException {try (Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {final Query query = new TermQuery(new Term("test", "foo"));- final Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));- assertEquals("this is a <B>foo</B> bar example",+ final Highlighter highlighter =+ new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));+ assertEquals(+ "this is a <B>foo</B> bar example",highlighter.getBestFragment(analyzer, "test", "this is a foo bar example"));assertNull(highlighter.getBestFragment(analyzer, "test", "this does not match"));}@@ -48,8 +49,10 @@ public class TestMisses extends LuceneTestCase {final BooleanQuery.Builder query = new BooleanQuery.Builder();query.add(new TermQuery(new Term("test", "foo")), Occur.MUST);query.add(new TermQuery(new Term("test", "bar")), Occur.MUST);- final Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query.build()));- assertEquals("this is a <B>foo</B> <B>bar</B> example",+ final Highlighter highlighter =+ new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query.build()));+ assertEquals(+ "this is a <B>foo</B> <B>bar</B> example",highlighter.getBestFragment(analyzer, "test", "this is a foo bar example"));assertNull(highlighter.getBestFragment(analyzer, "test", "this does not match"));}@@ -58,8 +61,10 @@ public class TestMisses extends LuceneTestCase {public void testPhraseQuery() throws IOException, InvalidTokenOffsetsException {try (Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {final PhraseQuery query = new PhraseQuery("test", "foo", "bar");- final Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));- assertEquals("this is a <B>foo</B> <B>bar</B> example",+ final Highlighter highlighter =+ new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));+ assertEquals(+ "this is a <B>foo</B> <B>bar</B> example",highlighter.getBestFragment(analyzer, "test", "this is a foo bar example"));assertNull(highlighter.getBestFragment(analyzer, "test", "this does not match"));}@@ -67,11 +72,18 @@ public class TestMisses extends LuceneTestCase {public void testSpanNearQuery() throws IOException, InvalidTokenOffsetsException {try (Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {- final Query query = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term("test", "foo")),- new SpanTermQuery(new Term("test", "bar"))}, 0, true);- final Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));- assertEquals("this is a <B>foo</B> <B>bar</B> example",+ final Query query =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("test", "foo")),+ new SpanTermQuery(new Term("test", "bar"))+ },+ 0,+ true);+ final Highlighter highlighter =+ new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));+ assertEquals(+ "this is a <B>foo</B> <B>bar</B> example",highlighter.getBestFragment(analyzer, "test", "this is a foo bar example"));assertNull(highlighter.getBestFragment(analyzer, "test", "this does not match"));}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestOffsetLimitTokenFilter.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestOffsetLimitTokenFilter.javaindex f0ce36c207d..aee0ccd78f8 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestOffsetLimitTokenFilter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestOffsetLimitTokenFilter.java@@ -17,42 +17,42 @@package org.apache.lucene.search.highlight;import java.io.StringReader;-import org.apache.lucene.analysis.*;public class TestOffsetLimitTokenFilter extends BaseTokenStreamTestCase {-+public void testFilter() throws Exception {- // we disable MockTokenizer checks because we will forcefully limit the+ // we disable MockTokenizer checks because we will forcefully limit the// tokenstream and call end() before incrementToken() returns false.- MockTokenizer stream = new MockTokenizer(- MockTokenizer.WHITESPACE, false);+ MockTokenizer stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));stream.setEnableChecks(false);OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);assertTokenStreamContents(filter, new String[] {"short", "toolong"});-+stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));stream.setEnableChecks(false);filter = new OffsetLimitTokenFilter(stream, 12);assertTokenStreamContents(filter, new String[] {"short", "toolong"});-+stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));stream.setEnableChecks(false);filter = new OffsetLimitTokenFilter(stream, 30);- assertTokenStreamContents(filter, new String[] {"short", "toolong",- "evenmuchlongertext"});-- checkOneTerm(new Analyzer() {-- @Override- public TokenStreamComponents createComponents(String fieldName) {- MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);- tokenizer.setEnableChecks(false);- return new TokenStreamComponents(tokenizer, new OffsetLimitTokenFilter(tokenizer, 10));- }- }, "llenges", "llenges");+ assertTokenStreamContents(filter, new String[] {"short", "toolong", "evenmuchlongertext"});++ checkOneTerm(+ new Analyzer() {++ @Override+ public TokenStreamComponents createComponents(String fieldName) {+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ tokenizer.setEnableChecks(false);+ return new TokenStreamComponents(tokenizer, new OffsetLimitTokenFilter(tokenizer, 10));+ }+ },+ "llenges",+ "llenges");}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestTokenSources.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestTokenSources.javaindex f23a65d0e20..5979e6797cb 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestTokenSources.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TestTokenSources.java@@ -16,11 +16,9 @@*/package org.apache.lucene.search.highlight;+import com.carrotsearch.randomizedtesting.annotations.Repeat;import java.io.IOException;import java.util.Arrays;--import com.carrotsearch.randomizedtesting.annotations.Repeat;-import org.apache.lucene.analysis.BaseTokenStreamTestCase;import org.apache.lucene.analysis.CannedTokenStream;import org.apache.lucene.analysis.MockAnalyzer;@@ -54,9 +52,9 @@ import org.apache.lucene.util.TestUtil;// LUCENE-2874-/** Tests {@link org.apache.lucene.search.highlight.TokenSources} and- * {@link org.apache.lucene.search.highlight.TokenStreamFromTermVector}- * indirectly from that.+/**+ * Tests {@link org.apache.lucene.search.highlight.TokenSources} and {@link+ * org.apache.lucene.search.highlight.TokenStreamFromTermVector} indirectly from that.*/public class TestTokenSources extends BaseTokenStreamTestCase {private static final String FIELD = "text";@@ -68,7 +66,8 @@ public class TestTokenSources extends BaseTokenStreamTestCase {private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);- private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);+ private final PositionIncrementAttribute positionIncrementAttribute =+ addAttribute(PositionIncrementAttribute.class);@Overridepublic boolean incrementToken() {@@ -78,23 +77,23 @@ public class TestTokenSources extends BaseTokenStreamTestCase {}clearAttributes();termAttribute.setEmpty().append(this.tokens[i]);- offsetAttribute.setOffset(this.tokens[i].startOffset(),- this.tokens[i].endOffset());- positionIncrementAttribute.setPositionIncrement(this.tokens[i]- .getPositionIncrement());+ offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i].endOffset());+ positionIncrementAttribute.setPositionIncrement(this.tokens[i].getPositionIncrement());return true;}@Overridepublic void reset() {this.i = -1;- this.tokens = new Token[] {- new Token("the", 0, 3),- new Token("{fox}", 0, 7),- new Token("fox", 4, 7),- new Token("did", 8, 11),- new Token("not", 12, 15),- new Token("jump", 16, 20)};+ this.tokens =+ new Token[] {+ new Token("the", 0, 3),+ new Token("{fox}", 0, 7),+ new Token("fox", 4, 7),+ new Token("did", 8, 11),+ new Token("not", 12, 15),+ new Token("jump", 16, 20)+ };this.tokens[1].setPositionIncrement(0);}}@@ -102,8 +101,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {public void testOverlapWithOffset() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(null));+ final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(null));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -119,36 +117,34 @@ public class TestTokenSources extends BaseTokenStreamTestCase {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);try {- final DisjunctionMaxQuery query = new DisjunctionMaxQuery(- Arrays.asList(- new SpanTermQuery(new Term(FIELD, "{fox}")),- new SpanTermQuery(new Term(FIELD, "fox"))),- 1);- // final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {- // new SpanTermQuery(new Term(FIELD, "{fox}")),- // new SpanTermQuery(new Term(FIELD, "fox")) }, 0, true);+ final DisjunctionMaxQuery query =+ new DisjunctionMaxQuery(+ Arrays.asList(+ new SpanTermQuery(new Term(FIELD, "{fox}")),+ new SpanTermQuery(new Term(FIELD, "fox"))),+ 1);+ // final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {+ // new SpanTermQuery(new Term(FIELD, "{fox}")),+ // new SpanTermQuery(new Term(FIELD, "fox")) }, 0, true);TopDocs hits = indexSearcher.search(query, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(query));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(query));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals("<B>the fox</B> did not jump",- highlighter.getBestFragment(tokenStream, TEXT));+ assertEquals("<B>the fox</B> did not jump", highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();}}- public void testOverlapWithPositionsAndOffset()- throws IOException, InvalidTokenOffsetsException {+ public void testOverlapWithPositionsAndOffset() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(null));+ final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(null));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -164,36 +160,34 @@ public class TestTokenSources extends BaseTokenStreamTestCase {try {assertEquals(1, indexReader.numDocs());final IndexSearcher indexSearcher = newSearcher(indexReader);- final DisjunctionMaxQuery query = new DisjunctionMaxQuery(- Arrays.asList(- new SpanTermQuery(new Term(FIELD, "{fox}")),- new SpanTermQuery(new Term(FIELD, "fox"))),- 1);+ final DisjunctionMaxQuery query =+ new DisjunctionMaxQuery(+ Arrays.asList(+ new SpanTermQuery(new Term(FIELD, "{fox}")),+ new SpanTermQuery(new Term(FIELD, "fox"))),+ 1);// final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {// new SpanTermQuery(new Term(FIELD, "{fox}")),// new SpanTermQuery(new Term(FIELD, "fox")) }, 0, true);TopDocs hits = indexSearcher.search(query, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(query));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(query));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals("<B>the fox</B> did not jump",- highlighter.getBestFragment(tokenStream, TEXT));+ assertEquals("<B>the fox</B> did not jump", highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();}}- public void testOverlapWithOffsetExactPhrase()- throws IOException, InvalidTokenOffsetsException {+ public void testOverlapWithOffsetExactPhrase() throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(null));+ final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(null));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -212,19 +206,22 @@ public class TestTokenSources extends BaseTokenStreamTestCase {// final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1);// query.add(new SpanTermQuery(new Term(FIELD, "{fox}")));// query.add(new SpanTermQuery(new Term(FIELD, "fox")));- final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD, "the")),- new SpanTermQuery(new Term(FIELD, "fox"))}, 0, true);+ final Query phraseQuery =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD, "the")), new SpanTermQuery(new Term(FIELD, "fox"))+ },+ 0,+ true);TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals("<B>the fox</B> did not jump",- highlighter.getBestFragment(tokenStream, TEXT));+ assertEquals("<B>the fox</B> did not jump", highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();@@ -235,8 +232,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {throws IOException, InvalidTokenOffsetsException {final String TEXT = "the fox did not jump";final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(null));+ final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(null));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -255,19 +251,22 @@ public class TestTokenSources extends BaseTokenStreamTestCase {// final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1);// query.add(new SpanTermQuery(new Term(FIELD, "the")));// query.add(new SpanTermQuery(new Term(FIELD, "fox")));- final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD, "the")),- new SpanTermQuery(new Term(FIELD, "fox"))}, 0, true);+ final Query phraseQuery =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD, "the")), new SpanTermQuery(new Term(FIELD, "fox"))+ },+ 0,+ true);TopDocs hits = indexSearcher.search(phraseQuery, 1);assertEquals(1, hits.totalHits.value);- final Highlighter highlighter = new Highlighter(- new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),- new QueryScorer(phraseQuery));+ final Highlighter highlighter =+ new Highlighter(+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery));final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);- assertEquals("<B>the fox</B> did not jump",- highlighter.getBestFragment(tokenStream, TEXT));+ assertEquals("<B>the fox</B> did not jump", highlighter.getBestFragment(tokenStream, TEXT));} finally {indexReader.close();directory.close();@@ -277,8 +276,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {public void testTermVectorWithoutOffsetsDoesntWork()throws IOException, InvalidTokenOffsetsException {final Directory directory = newDirectory();- final IndexWriter indexWriter = new IndexWriter(directory,- newIndexWriterConfig(null));+ final IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig(null));try {final Document document = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);@@ -296,8 +294,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {final TokenStream tokenStream =TokenSources.getTermVectorTokenStreamOrNull(FIELD, indexReader.getTermVectors(0), -1);assertNull(tokenStream);- }- finally {+ } finally {indexReader.close();directory.close();}@@ -305,10 +302,12 @@ public class TestTokenSources extends BaseTokenStreamTestCase {int curOffset;- /** Just make a token with the text, and set the payload- * to the text as well. Offsets increment "naturally". */+ /**+ * Just make a token with the text, and set the payload to the text as well. Offsets increment+ * "naturally".+ */private Token getToken(String text) {- Token t = new Token(text, curOffset, curOffset+text.length());+ Token t = new Token(text, curOffset, curOffset + text.length());t.setPayload(new BytesRef(text));curOffset++;return t;@@ -326,22 +325,19 @@ public class TestTokenSources extends BaseTokenStreamTestCase {curOffset = 0;- Token[] tokens = new Token[] {- getToken("foxes"),- getToken("can"),- getToken("jump"),- getToken("high")- };+ Token[] tokens =+ new Token[] {getToken("foxes"), getToken("can"), getToken("jump"), getToken("high")};Document doc = new Document();doc.add(new Field("field", new CannedTokenStream(tokens), myFieldType));writer.addDocument(doc);-+IndexReader reader = writer.getReader();writer.close();assertEquals(1, reader.numDocs());- TokenStream ts = TokenSources.getTermVectorTokenStreamOrNull("field", reader.getTermVectors(0), -1);+ TokenStream ts =+ TokenSources.getTermVectorTokenStreamOrNull("field", reader.getTermVectors(0), -1);CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);PositionIncrementAttribute posIncAtt = ts.getAttribute(PositionIncrementAttribute.class);@@ -349,7 +345,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);ts.reset();- for(Token token : tokens) {+ for (Token token : tokens) {assertTrue(ts.incrementToken());assertEquals(token.toString(), termAtt.toString());assertEquals(token.getPositionIncrement(), posIncAtt.getPositionIncrement());@@ -365,7 +361,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {}@Repeat(iterations = 10)- //@Seed("947083AB20AB2D4F")+ // @Seed("947083AB20AB2D4F")public void testRandomizedRoundTrip() throws Exception {final int distinct = TestUtil.nextInt(random(), 1, 10);@@ -377,8 +373,9 @@ public class TestTokenSources extends BaseTokenStreamTestCase {}final BaseTermVectorsFormatTestCase.RandomTokenStream rTokenStream =- new BaseTermVectorsFormatTestCase.RandomTokenStream(TestUtil.nextInt(random(), 1, 10), terms, termBytes);- //check to see if the token streams might have non-deterministic testable result+ new BaseTermVectorsFormatTestCase.RandomTokenStream(+ TestUtil.nextInt(random(), 1, 10), terms, termBytes);+ // check to see if the token streams might have non-deterministic testable resultfinal boolean storeTermVectorPositions = random().nextBoolean();final int[] startOffsets = rTokenStream.getStartOffsets();final int[] positionsIncrements = rTokenStream.getPositionsIncrements();@@ -386,19 +383,24 @@ public class TestTokenSources extends BaseTokenStreamTestCase {if (storeTermVectorPositions && positionsIncrements[i] != 0) {continue;}- //TODO should RandomTokenStream ensure endOffsets for tokens at same position and same startOffset are greater+ // TODO should RandomTokenStream ensure endOffsets for tokens at same position and same+ // startOffset are greater// than previous token's endOffset? That would increase the testable possibilities.- if (startOffsets[i] == startOffsets[i-1]) {+ if (startOffsets[i] == startOffsets[i - 1]) {if (VERBOSE)- System.out.println("Skipping test because can't easily validate random token-stream is correct.");+ System.out.println(+ "Skipping test because can't easily validate random token-stream is correct.");rTokenStream.close();return;}}- //sanity check itself- assertTokenStreamContents(rTokenStream,- rTokenStream.getTerms(), rTokenStream.getStartOffsets(), rTokenStream.getEndOffsets(),+ // sanity check itself+ assertTokenStreamContents(+ rTokenStream,+ rTokenStream.getTerms(),+ rTokenStream.getStartOffsets(),+ rTokenStream.getEndOffsets(),rTokenStream.getPositionsIncrements());Directory dir = newDirectory();@@ -407,7 +409,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {myFieldType.setStoreTermVectors(true);myFieldType.setStoreTermVectorOffsets(true);myFieldType.setStoreTermVectorPositions(storeTermVectorPositions);- //payloads require positions; it will throw an error otherwise+ // payloads require positions; it will throw an error otherwisemyFieldType.setStoreTermVectorPayloads(storeTermVectorPositions && random().nextBoolean());Document doc = new Document();@@ -421,15 +423,18 @@ public class TestTokenSources extends BaseTokenStreamTestCase {TokenStream vectorTokenStream =TokenSources.getTermVectorTokenStreamOrNull("field", reader.getTermVectors(0), -1);- //sometimes check payloads+ // sometimes check payloadsPayloadAttribute payloadAttribute = null;if (myFieldType.storeTermVectorPayloads() && usually()) {payloadAttribute = vectorTokenStream.addAttribute(PayloadAttribute.class);}- assertTokenStreamContents(vectorTokenStream,- rTokenStream.getTerms(), rTokenStream.getStartOffsets(), rTokenStream.getEndOffsets(),+ assertTokenStreamContents(+ vectorTokenStream,+ rTokenStream.getTerms(),+ rTokenStream.getStartOffsets(),+ rTokenStream.getEndOffsets(),myFieldType.storeTermVectorPositions() ? rTokenStream.getPositionsIncrements() : null);- //test payloads+ // test payloadsif (payloadAttribute != null) {vectorTokenStream.reset();for (int i = 0; vectorTokenStream.incrementToken(); i++) {@@ -451,7 +456,8 @@ public class TestTokenSources extends BaseTokenStreamTestCase {Directory dir = newDirectory();MockAnalyzer analyzer = new MockAnalyzer(random());- analyzer.setEnableChecks(false);//we don't necessarily consume the whole stream because of limiting by startOffset+ analyzer.setEnableChecks(+ false); // we don't necessarily consume the whole stream because of limiting by startOffsetDocument doc = new Document();final String TEXT = " f gg h";doc.add(new Field("fld_tv", analyzer.tokenStream("fooFld", TEXT), tvFieldType));@@ -465,10 +471,12 @@ public class TestTokenSources extends BaseTokenStreamTestCase {try {Fields tvFields = reader.getTermVectors(0);for (int maxStartOffset = -1; maxStartOffset <= TEXT.length(); maxStartOffset++) {- TokenStream tvStream = TokenSources.getTokenStream("fld_tv", tvFields, TEXT, analyzer, maxStartOffset);- TokenStream anaStream = TokenSources.getTokenStream("fld_notv", tvFields, TEXT, analyzer, maxStartOffset);+ TokenStream tvStream =+ TokenSources.getTokenStream("fld_tv", tvFields, TEXT, analyzer, maxStartOffset);+ TokenStream anaStream =+ TokenSources.getTokenStream("fld_notv", tvFields, TEXT, analyzer, maxStartOffset);- //assert have same tokens, none of which has a start offset > maxStartOffset+ // assert have same tokens, none of which has a start offset > maxStartOffsetfinal OffsetAttribute tvOffAtt = tvStream.addAttribute(OffsetAttribute.class);final OffsetAttribute anaOffAtt = anaStream.addAttribute(OffsetAttribute.class);tvStream.reset();@@ -476,8 +484,7 @@ public class TestTokenSources extends BaseTokenStreamTestCase {while (tvStream.incrementToken()) {assertTrue(anaStream.incrementToken());assertEquals(tvOffAtt.startOffset(), anaOffAtt.startOffset());- if (maxStartOffset >= 0)- assertTrue(tvOffAtt.startOffset() <= maxStartOffset);+ if (maxStartOffset >= 0) assertTrue(tvOffAtt.startOffset() <= maxStartOffset);}assertTrue(anaStream.incrementToken() == false);tvStream.end();@@ -490,8 +497,6 @@ public class TestTokenSources extends BaseTokenStreamTestCase {reader.close();}--dir.close();}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/TestHighlightCustomQuery.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/TestHighlightCustomQuery.javaindex 1011efe85fd..367b79620f3 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/TestHighlightCustomQuery.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/TestHighlightCustomQuery.java@@ -20,7 +20,6 @@ import java.io.IOException;import java.util.Collections;import java.util.Map;import java.util.Objects;-import org.apache.lucene.analysis.CannedTokenStream;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenFilter;@@ -42,15 +41,14 @@ import org.apache.lucene.search.highlight.WeightedSpanTermExtractor;import org.apache.lucene.util.LuceneTestCase;/**- * Tests the extensibility of {@link WeightedSpanTermExtractor} and- * {@link QueryScorer} in a user defined package+ * Tests the extensibility of {@link WeightedSpanTermExtractor} and {@link QueryScorer} in a user+ * defined package*/public class TestHighlightCustomQuery extends LuceneTestCase {private static final String FIELD_NAME = "contents";- public void testHighlightCustomQuery() throws IOException,- InvalidTokenOffsetsException {+ public void testHighlightCustomQuery() throws IOException, InvalidTokenOffsetsException {String s1 = "I call our world Flatland, not because we call it so,";// Verify that a query against the default field results in text being@@ -61,12 +59,11 @@ public class TestHighlightCustomQuery extends LuceneTestCase {String expected = "I call our <B>world</B> Flatland, not because we call it so,";String observed = highlightField(q, "SOME_FIELD_NAME", s1);- if (VERBOSE)- System.out.println("Expected: \"" + expected + "\n" + "Observed: \""- + observed);+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);assertEquals("Query in the default field results in text for *ANY* field being highlighted",- expected, observed);+ expected,+ observed);// Verify that a query against a named field does not result in any// highlighting@@ -77,50 +74,48 @@ public class TestHighlightCustomQuery extends LuceneTestCase {expected = s1;observed = highlightField(q, FIELD_NAME, s1);- if (VERBOSE)- System.out.println("Expected: \"" + expected + "\n" + "Observed: \""- + observed);+ if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);assertEquals("Query in a named field does not result in highlighting when that field isn't in the query",- s1, highlightField(q, FIELD_NAME, s1));-+ s1,+ highlightField(q, FIELD_NAME, s1));}public void testHighlightKnownQuery() throws IOException {- WeightedSpanTermExtractor extractor = new WeightedSpanTermExtractor() {- @Override- protected void extractUnknownQuery(Query query, Map<String,WeightedSpanTerm> terms) throws IOException {- terms.put("foo", new WeightedSpanTerm(3, "foo"));- }- };- Map<String,WeightedSpanTerm> terms = extractor.getWeightedSpanTerms(- new TermQuery(new Term("bar", "quux")), 3, new CannedTokenStream());+ WeightedSpanTermExtractor extractor =+ new WeightedSpanTermExtractor() {+ @Override+ protected void extractUnknownQuery(Query query, Map<String, WeightedSpanTerm> terms)+ throws IOException {+ terms.put("foo", new WeightedSpanTerm(3, "foo"));+ }+ };+ Map<String, WeightedSpanTerm> terms =+ extractor.getWeightedSpanTerms(+ new TermQuery(new Term("bar", "quux")), 3, new CannedTokenStream());// no fooassertEquals(Collections.singleton("quux"), terms.keySet());}- /**- * This method intended for use with- * <code>testHighlightingWithDefaultField()</code>- */- private String highlightField(Query query, String fieldName,- String text) throws IOException, InvalidTokenOffsetsException {- try (MockAnalyzer mockAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE,true,- MockTokenFilter.ENGLISH_STOPSET); TokenStream tokenStream = mockAnalyzer.tokenStream(fieldName, text)) {+ /** This method intended for use with <code>testHighlightingWithDefaultField()</code> */+ private String highlightField(Query query, String fieldName, String text)+ throws IOException, InvalidTokenOffsetsException {+ try (MockAnalyzer mockAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);+ TokenStream tokenStream = mockAnalyzer.tokenStream(fieldName, text)) {// Assuming "<B>", "</B>" used to highlightSimpleHTMLFormatter formatter = new SimpleHTMLFormatter();MyQueryScorer scorer = new MyQueryScorer(query, fieldName, FIELD_NAME);Highlighter highlighter = new Highlighter(formatter, scorer);highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));- String rv = highlighter.getBestFragments(tokenStream, text, 1,- "(FIELD TEXT TRUNCATED)");+ String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");return rv.length() == 0 ? text : rv;}}- public static class MyWeightedSpanTermExtractor extends- WeightedSpanTermExtractor {+ public static class MyWeightedSpanTermExtractor extends WeightedSpanTermExtractor {public MyWeightedSpanTermExtractor() {super();@@ -131,8 +126,8 @@ public class TestHighlightCustomQuery extends LuceneTestCase {}@Override- protected void extractUnknownQuery(Query query,- Map<String, WeightedSpanTerm> terms) throws IOException {+ protected void extractUnknownQuery(Query query, Map<String, WeightedSpanTerm> terms)+ throws IOException {float boost = 1f;while (query instanceof BoostQuery) {BoostQuery bq = (BoostQuery) query;@@ -143,7 +138,6 @@ public class TestHighlightCustomQuery extends LuceneTestCase {extractWeightedTerms(terms, new TermQuery(((CustomQuery) query).term), boost);}}-}public static class MyQueryScorer extends QueryScorer {@@ -154,10 +148,10 @@ public class TestHighlightCustomQuery extends LuceneTestCase {@Overrideprotected WeightedSpanTermExtractor newTermExtractor(String defaultField) {- return defaultField == null ? new MyWeightedSpanTermExtractor()+ return defaultField == null+ ? new MyWeightedSpanTermExtractor(): new MyWeightedSpanTermExtractor(defaultField);}-}public static class CustomQuery extends Query {@@ -189,8 +183,7 @@ public class TestHighlightCustomQuery extends LuceneTestCase {@Overridepublic boolean equals(Object other) {- return sameClassAs(other) &&- Objects.equals(term, ((CustomQuery) other).term);+ return sameClassAs(other) && Objects.equals(term, ((CustomQuery) other).term);}}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AnalyzerWithGaps.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AnalyzerWithGaps.javaindex 3009c91bbe5..8697882b519 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AnalyzerWithGaps.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AnalyzerWithGaps.java@@ -19,9 +19,7 @@ package org.apache.lucene.search.matchhighlight;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;-/**- * An analyzer for tests that has a predefined offset and position gap.- */+/** An analyzer for tests that has a predefined offset and position gap. */class AnalyzerWithGaps extends DelegatingAnalyzerWrapper {private final Analyzer delegate;private final int offsetGap;@@ -48,4 +46,4 @@ class AnalyzerWithGaps extends DelegatingAnalyzerWrapper {public int getPositionIncrementGap(String fieldName) {return positionGap;}-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AsciiMatchRangeHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AsciiMatchRangeHighlighter.javaindex fa01af8d2e4..d56fd305f51 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AsciiMatchRangeHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/AsciiMatchRangeHighlighter.java@@ -16,17 +16,14 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.analysis.Analyzer;-import org.apache.lucene.document.Document;-import java.util.ArrayList;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.document.Document;-/**- * A simple ASCII match range highlighter for tests.- */+/** A simple ASCII match range highlighter for tests. */final class AsciiMatchRangeHighlighter {private final Analyzer analyzer;private final PassageFormatter passageFormatter;@@ -41,7 +38,8 @@ final class AsciiMatchRangeHighlighter {this.analyzer = analyzer;}- public Map<String, List<String>> apply(Document document, Map<String, List<OffsetRange>> fieldHighlights) {+ public Map<String, List<String>> apply(+ Document document, Map<String, List<OffsetRange>> fieldHighlights) {ArrayList<OffsetRange> valueRanges = new ArrayList<>();Map<String, List<String>> fieldSnippets = new LinkedHashMap<>();diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/IndexBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/IndexBuilder.javaindex 2a6e783f80e..c409c750ccb 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/IndexBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/IndexBuilder.java@@ -17,6 +17,10 @@package org.apache.lucene.search.matchhighlight;import com.carrotsearch.randomizedtesting.RandomizedTest;+import java.io.IOException;+import java.util.ArrayList;+import java.util.function.BiFunction;+import java.util.function.Consumer;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -32,14 +36,8 @@ import org.apache.lucene.store.ByteBuffersDirectory;import org.apache.lucene.store.Directory;import org.apache.lucene.util.IOUtils;-import java.io.IOException;-import java.util.ArrayList;-import java.util.function.BiFunction;-import java.util.function.Consumer;-/**- * Utility class for building an ephemeral document index- * and running a block of code on its reader.+ * Utility class for building an ephemeral document index and running a block of code on its reader.*/class IndexBuilder {public static final String FLD_ID = "id";@@ -69,9 +67,10 @@ class IndexBuilder {}public IndexBuilder doc(String field, String... values) {- return doc(fields -> {- fields.add(field, values);- });+ return doc(+ fields -> {+ fields.add(field, values);+ });}public IndexBuilder doc(Consumer<DocFields> fields) {@@ -83,7 +82,8 @@ class IndexBuilder {return this;}- public IndexBuilder build(Analyzer analyzer, IOUtils.IOConsumer<DirectoryReader> block) throws IOException {+ public IndexBuilder build(Analyzer analyzer, IOUtils.IOConsumer<DirectoryReader> block)+ throws IOException {IndexWriterConfig config = new IndexWriterConfig(analyzer);config.setIndexSort(new Sort(new SortField(FLD_SORT_ORDER, SortField.Type.LONG)));try (Directory directory = new ByteBuffersDirectory()) {diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/MissingAnalyzer.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/MissingAnalyzer.javaindex 6ff206732ce..a6147d36a61 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/MissingAnalyzer.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/MissingAnalyzer.java@@ -16,9 +16,8 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.analysis.Analyzer;-import java.io.Reader;+import org.apache.lucene.analysis.Analyzer;/** An {@link Analyzer} that throws a runtime exception when used for anything. */final class MissingAnalyzer extends Analyzer {diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.javaindex d1acf982541..9b3ebfcf3ba 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java@@ -17,6 +17,20 @@package org.apache.lucene.search.matchhighlight;import com.carrotsearch.randomizedtesting.RandomizedTest;+import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collection;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Objects;+import java.util.Set;+import java.util.function.Function;+import java.util.stream.Collectors;+import java.util.stream.Stream;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;@@ -47,21 +61,6 @@ import org.hamcrest.Matchers;import org.junit.Before;import org.junit.Test;-import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collection;-import java.util.Collections;-import java.util.HashMap;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.Objects;-import java.util.Set;-import java.util.function.Function;-import java.util.stream.Collectors;-import java.util.stream.Stream;-public class TestMatchHighlighter extends LuceneTestCase {private static final String FLD_ID = "id";private static final String FLD_TEXT1 = "text1";@@ -77,30 +76,36 @@ public class TestMatchHighlighter extends LuceneTestCase {TYPE_TEXT_POSITIONS = TextField.TYPE_STORED;TYPE_TEXT_POSITIONS_OFFSETS = new FieldType(TextField.TYPE_STORED);- TYPE_TEXT_POSITIONS_OFFSETS.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);+ TYPE_TEXT_POSITIONS_OFFSETS.setIndexOptions(+ IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);TYPE_TEXT_POSITIONS_OFFSETS.freeze();Map<String, Analyzer> fieldAnalyzers = new HashMap<>();// Create an analyzer with some synonyms, just to showcase them.- SynonymMap synonymMap = buildSynonymMap(new String[][]{- {"moon\u0000shine", "firewater"},- {"firewater", "moon\u0000shine"},- });+ SynonymMap synonymMap =+ buildSynonymMap(+ new String[][] {+ {"moon\u0000shine", "firewater"},+ {"firewater", "moon\u0000shine"},+ });// Make a non-empty offset gap so that break iterator doesn't go haywire on multivalues// glued together.final int offsetGap = RandomizedTest.randomIntBetween(1, 2);- final int positionGap = RandomizedTest.randomFrom(new int[]{0, 1, 100});+ final int positionGap = RandomizedTest.randomFrom(new int[] {0, 1, 100});Analyzer synonymsAnalyzer =- new AnalyzerWithGaps(offsetGap, positionGap, new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new WhitespaceTokenizer();- TokenStream tokenStream = new SynonymGraphFilter(tokenizer, synonymMap, true);- return new TokenStreamComponents(tokenizer, tokenStream);- }- });+ new AnalyzerWithGaps(+ offsetGap,+ positionGap,+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new WhitespaceTokenizer();+ TokenStream tokenStream = new SynonymGraphFilter(tokenizer, synonymMap, true);+ return new TokenStreamComponents(tokenizer, tokenStream);+ }+ });fieldAnalyzers.put(FLD_TEXT1, synonymsAnalyzer);fieldAnalyzers.put(FLD_TEXT2, synonymsAnalyzer);@@ -122,91 +127,109 @@ public class TestMatchHighlighter extends LuceneTestCase {new IndexBuilder(this::toField).doc(FLD_TEXT1, "foo bar baz").doc(FLD_TEXT1, "bar foo baz")- .doc(fields -> {- fields.add(FLD_TEXT1, "Very long content but not matching anything.");- fields.add(FLD_TEXT2, "no foo but bar");- })- .build(analyzer, reader -> {- Query query = new BooleanQuery.Builder()- .add(new TermQuery(new Term(FLD_TEXT1, "foo")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term(FLD_TEXT2, "bar")), BooleanClause.Occur.SHOULD)- .build();-- // In the most basic scenario, we run a search against a query, retrieve- // top docs...- IndexSearcher searcher = new IndexSearcher(reader);- Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.- TopDocs topDocs = searcher.search(query, 10, sortOrder);-- // ...and would want a fixed set of fields from those documents, some of them- // possibly highlighted if they matched the query.- //- // This configures the highlighter so that the FLD_ID field is always returned verbatim,- // and FLD_TEXT1 is returned *only if it contained a query match*.- MatchHighlighter highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))- .appendFieldHighlighter(FieldValueHighlighters.highlighted(- 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- // Note document field highlights are a stream over documents in topDocs. In the remaining code we will just- // collect them on the fly into a preformatted string.- Stream<MatchHighlighter.DocHighlights> highlights = highlighter.highlight(topDocs, query);- assertHighlights(toDocList(highlights),+ .doc(+ fields -> {+ fields.add(FLD_TEXT1, "Very long content but not matching anything.");+ fields.add(FLD_TEXT2, "no foo but bar");+ })+ .build(+ analyzer,+ reader -> {+ Query query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term(FLD_TEXT1, "foo")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term(FLD_TEXT2, "bar")), BooleanClause.Occur.SHOULD)+ .build();++ // In the most basic scenario, we run a search against a query, retrieve+ // top docs...+ IndexSearcher searcher = new IndexSearcher(reader);+ Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.+ TopDocs topDocs = searcher.search(query, 10, sortOrder);++ // ...and would want a fixed set of fields from those documents, some of them+ // possibly highlighted if they matched the query.+ //+ // This configures the highlighter so that the FLD_ID field is always returned+ // verbatim,+ // and FLD_TEXT1 is returned *only if it contained a query match*.+ MatchHighlighter highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))+ .appendFieldHighlighter(+ FieldValueHighlighters.highlighted(+ 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ // Note document field highlights are a stream over documents in topDocs. In the+ // remaining code we will just+ // collect them on the fly into a preformatted string.+ Stream<MatchHighlighter.DocHighlights> highlights =+ highlighter.highlight(topDocs, query);+ assertHighlights(+ toDocList(highlights)," 0. id: 0"," text1: >foo< bar baz"," 1. id: 1"," text1: bar >foo< baz"," 2. id: 2");- // In a more realistic use case, you'd want to show the value of a given field *regardless* of whether it- // contained a highlight or not -- it is odd that document "id: 2" above doesn't have the 'text1' field- // shown because that field wasn't part of the query match.- //- // Let's say the field is also potentially long; if it contains a match,- // we would want to display the contextual snippet surrounding that match. If it does not contain any- // matches, we would want to display its content up to a given number of characters (lead lines).- //- // Let's do this by adding an appropriate field highlighter on FLD_TEXT1.- highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))- .appendFieldHighlighter(FieldValueHighlighters.highlighted(- 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))- .appendFieldHighlighter(FieldValueHighlighters.maxLeadingCharacters(10, "...", Set.of(FLD_TEXT1)))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- assertHighlights(toDocList(highlighter.highlight(topDocs, query)),- " 0. id: 0",- " text1: >foo< bar baz",- " 1. id: 1",- " text1: bar >foo< baz",- " 2. id: 2",- " text1: Very long...");-- // Field highlighters can apply to multiple fields and be chained for convenience.- // For example, this defines a combined highlighter over both FLD_TEXT1 and FLD_TEXT2.- Set<String> fields = Set.of(FLD_TEXT1, FLD_TEXT2);- MatchHighlighter.FieldValueHighlighter highlightedOrAbbreviated =- FieldValueHighlighters.highlighted(80 * 3, 1, new PassageFormatter("...", ">", "<"), fields::contains)- .or(FieldValueHighlighters.maxLeadingCharacters(10, "...", fields));-- highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))- .appendFieldHighlighter(highlightedOrAbbreviated)- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- assertHighlights(toDocList(highlighter.highlight(topDocs, query)),- " 0. id: 0",- " text1: >foo< bar baz",- " 1. id: 1",- " text1: bar >foo< baz",- " 2. id: 2",- " text1: Very long...",- " text2: no foo but >bar<");- });+ // In a more realistic use case, you'd want to show the value of a given field+ // *regardless* of whether it+ // contained a highlight or not -- it is odd that document "id: 2" above doesn't have+ // the 'text1' field+ // shown because that field wasn't part of the query match.+ //+ // Let's say the field is also potentially long; if it contains a match,+ // we would want to display the contextual snippet surrounding that match. If it does+ // not contain any+ // matches, we would want to display its content up to a given number of characters+ // (lead lines).+ //+ // Let's do this by adding an appropriate field highlighter on FLD_TEXT1.+ highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))+ .appendFieldHighlighter(+ FieldValueHighlighters.highlighted(+ 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))+ .appendFieldHighlighter(+ FieldValueHighlighters.maxLeadingCharacters(10, "...", Set.of(FLD_TEXT1)))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ assertHighlights(+ toDocList(highlighter.highlight(topDocs, query)),+ " 0. id: 0",+ " text1: >foo< bar baz",+ " 1. id: 1",+ " text1: bar >foo< baz",+ " 2. id: 2",+ " text1: Very long...");++ // Field highlighters can apply to multiple fields and be chained for convenience.+ // For example, this defines a combined highlighter over both FLD_TEXT1 and FLD_TEXT2.+ Set<String> fields = Set.of(FLD_TEXT1, FLD_TEXT2);+ MatchHighlighter.FieldValueHighlighter highlightedOrAbbreviated =+ FieldValueHighlighters.highlighted(+ 80 * 3, 1, new PassageFormatter("...", ">", "<"), fields::contains)+ .or(FieldValueHighlighters.maxLeadingCharacters(10, "...", fields));++ highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(FieldValueHighlighters.verbatimValue(FLD_ID))+ .appendFieldHighlighter(highlightedOrAbbreviated)+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ assertHighlights(+ toDocList(highlighter.highlight(topDocs, query)),+ " 0. id: 0",+ " text1: >foo< bar baz",+ " 1. id: 1",+ " text1: bar >foo< baz",+ " 2. id: 2",+ " text1: Very long...",+ " text2: no foo but >bar<");+ });}@Test@@ -215,24 +238,29 @@ public class TestMatchHighlighter extends LuceneTestCase {// Synonyms defined in the constructor of this class.new IndexBuilder(this::toField).doc(FLD_TEXT1, "Where the moon shine falls, firewater flows.")- .build(analyzer, reader -> {- IndexSearcher searcher = new IndexSearcher(reader);- Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.-- MatchHighlighter highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(FieldValueHighlighters.highlighted(- 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- Query query = new TermQuery(new Term(FLD_TEXT1, "firewater"));- assertHighlights(toDocList(highlighter.highlight(searcher.search(query, 10, sortOrder), query)),- "0. text1: Where the >moon shine< falls, >firewater< flows.");-- query = new PhraseQuery(FLD_TEXT1, "moon", "shine");- assertHighlights(toDocList(highlighter.highlight(searcher.search(query, 10, sortOrder), query)),- "0. text1: Where the >moon shine< falls, >firewater< flows.");- });+ .build(+ analyzer,+ reader -> {+ IndexSearcher searcher = new IndexSearcher(reader);+ Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.++ MatchHighlighter highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(+ FieldValueHighlighters.highlighted(+ 80 * 3, 1, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ Query query = new TermQuery(new Term(FLD_TEXT1, "firewater"));+ assertHighlights(+ toDocList(highlighter.highlight(searcher.search(query, 10, sortOrder), query)),+ "0. text1: Where the >moon shine< falls, >firewater< flows.");++ query = new PhraseQuery(FLD_TEXT1, "moon", "shine");+ assertHighlights(+ toDocList(highlighter.highlight(searcher.search(query, 10, sortOrder), query)),+ "0. text1: Where the >moon shine< falls, >firewater< flows.");+ });}@Test@@ -242,88 +270,105 @@ public class TestMatchHighlighter extends LuceneTestCase {//// In this example we will build a custom field highlighting handler that// highlights matches over a multivalued field, shows that field's values if it received- // no matches and limits the number of values displayed to at most 2 (with an appropriate message).+ // no matches and limits the number of values displayed to at most 2 (with an appropriate+ // message).new IndexBuilder(this::toField)// Just one document, one field, four values..doc(FLD_TEXT1, "foo bar", "bar foo baz", "bar baz foo", "baz baz baz")- .build(analyzer, reader -> {- IndexSearcher searcher = new IndexSearcher(reader);- Sort sortOrder = Sort.INDEXORDER;-- // Let's start with the simple predefined highlighter so that the field's value shows- // and is highlighted when it was part of the hit.- MatchHighlighter.FieldValueHighlighter highlighted = FieldValueHighlighters.highlighted(- 80 * 3, 2, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals);- MatchHighlighter highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(highlighted)- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- Query query = new TermQuery(new Term(FLD_TEXT1, "foo"));- TopDocs topDocs = searcher.search(query, 10, sortOrder);-- // Note the highlighter is configured with at most 2 snippets so the match on the- // third value ("bar baz foo") is omitted. Ellipsis isn't inserted too because- // values are displayed in full.- assertHighlights(toDocList(highlighter.highlight(topDocs, query)),- "0. text1: >foo< bar, bar >foo< baz");-- // So the above works fine if the field received a match but omits it otherwise. We can- // force the display of this field by chaining with verbatim value highlighter:- highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(highlighted.or(FieldValueHighlighters.verbatimValue(FLD_TEXT1)))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- assertHighlights(toDocList(highlighter.highlight(topDocs, new MatchAllDocsQuery())),- "0. text1: foo bar, bar foo baz, bar baz foo, baz baz baz");-- // But this is not exactly what we'd like because we want to limit the display of values to the first two.- // Let's just write a custom field highlighter handler that does it.- class AtMostNValuesHighlighter implements MatchHighlighter.FieldValueHighlighter {- private final String field;- private final int limit;-- AtMostNValuesHighlighter(String field, int limit) {- this.field = field;- this.limit = limit;- }-- @Override- public boolean isApplicable(String field, boolean hasMatches) {- return Objects.equals(field, this.field);- }-- @Override- public List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<MatchHighlighter.QueryOffsetRange> matchOffsets) {- if (values.length <= limit) {- return Arrays.asList(values);- } else {- List<String> collected = Stream.of(values).limit(limit).collect(Collectors.toList());- int remaining = values.length - collected.size();- collected.add(String.format(Locale.ROOT, "[%d omitted]", remaining));- return collected;+ .build(+ analyzer,+ reader -> {+ IndexSearcher searcher = new IndexSearcher(reader);+ Sort sortOrder = Sort.INDEXORDER;++ // Let's start with the simple predefined highlighter so that the field's value shows+ // and is highlighted when it was part of the hit.+ MatchHighlighter.FieldValueHighlighter highlighted =+ FieldValueHighlighters.highlighted(+ 80 * 3, 2, new PassageFormatter("...", ">", "<"), FLD_TEXT1::equals);+ MatchHighlighter highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(highlighted)+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ Query query = new TermQuery(new Term(FLD_TEXT1, "foo"));+ TopDocs topDocs = searcher.search(query, 10, sortOrder);++ // Note the highlighter is configured with at most 2 snippets so the match on the+ // third value ("bar baz foo") is omitted. Ellipsis isn't inserted too because+ // values are displayed in full.+ assertHighlights(+ toDocList(highlighter.highlight(topDocs, query)),+ "0. text1: >foo< bar, bar >foo< baz");++ // So the above works fine if the field received a match but omits it otherwise. We+ // can+ // force the display of this field by chaining with verbatim value highlighter:+ highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(+ highlighted.or(FieldValueHighlighters.verbatimValue(FLD_TEXT1)))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ assertHighlights(+ toDocList(highlighter.highlight(topDocs, new MatchAllDocsQuery())),+ "0. text1: foo bar, bar foo baz, bar baz foo, baz baz baz");++ // But this is not exactly what we'd like because we want to limit the display of+ // values to the first two.+ // Let's just write a custom field highlighter handler that does it.+ class AtMostNValuesHighlighter implements MatchHighlighter.FieldValueHighlighter {+ private final String field;+ private final int limit;++ AtMostNValuesHighlighter(String field, int limit) {+ this.field = field;+ this.limit = limit;+ }++ @Override+ public boolean isApplicable(String field, boolean hasMatches) {+ return Objects.equals(field, this.field);+ }++ @Override+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ if (values.length <= limit) {+ return Arrays.asList(values);+ } else {+ List<String> collected =+ Stream.of(values).limit(limit).collect(Collectors.toList());+ int remaining = values.length - collected.size();+ collected.add(String.format(Locale.ROOT, "[%d omitted]", remaining));+ return collected;+ }+ }++ @Override+ public Collection<String> alwaysFetchedFields() {+ return Collections.singleton(field);+ }}- }-- @Override- public Collection<String> alwaysFetchedFields() {- return Collections.singleton(field);- }- }-- // We can now chain it as usual and contemplate the result.- highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(highlighted.or(new AtMostNValuesHighlighter(FLD_TEXT1, 2)))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- assertHighlights(toDocList(highlighter.highlight(topDocs, query)),- "0. text1: >foo< bar, bar >foo< baz");- assertHighlights(toDocList(highlighter.highlight(topDocs, new MatchAllDocsQuery())),- "0. text1: foo bar, bar foo baz, [2 omitted]");- });++ // We can now chain it as usual and contemplate the result.+ highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(+ highlighted.or(new AtMostNValuesHighlighter(FLD_TEXT1, 2)))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ assertHighlights(+ toDocList(highlighter.highlight(topDocs, query)),+ "0. text1: >foo< bar, bar >foo< baz");+ assertHighlights(+ toDocList(highlighter.highlight(topDocs, new MatchAllDocsQuery())),+ "0. text1: foo bar, bar foo baz, [2 omitted]");+ });}@Test@@ -336,95 +381,124 @@ public class TestMatchHighlighter extends LuceneTestCase {new IndexBuilder(this::toField).doc(FLD_TEXT1, "foo bar baz").doc(FLD_TEXT1, "foo baz bar")- .build(analyzer, reader -> {- // Let's start with the two queries. The first one will be an unordered- // query for (foo, baz) with a max gap of 1; let's use intervals for this.- Query q1 = new IntervalQuery(FLD_TEXT1,- Intervals.maxgaps(1,- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("baz"))));-- // The second one will be a simpler term query for "bar".- Query q2 = new TermQuery(new Term(FLD_TEXT1, "bar"));-- // Let's fetch matching documents by combining the two into a Boolean query.- Query query = new BooleanQuery.Builder()- .add(q1, BooleanClause.Occur.SHOULD)- .add(q2, BooleanClause.Occur.SHOULD)- .build();-- IndexSearcher searcher = new IndexSearcher(reader);- Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.- TopDocs topDocs = searcher.search(query, 10, sortOrder);-- // If we use the "regular" highlighter, the result will be slightly odd: a nested- // highlight over "bar" within the first match. Also, you can't distinguish which of the sub-queries- // caused which highlight marker... but if it were HTML then you could give the span- // some semi-translucent background and layered matches would be visible.- MatchHighlighter highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(FieldValueHighlighters.highlighted(- 80 * 3, 1, new PassageFormatter("...", "<span>", "</span>"), FLD_TEXT1::equals))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- assertHighlights(toDocList(highlighter.highlight(topDocs, query)),- "0. text1: <span>foo <span>bar</span> baz</span>",- "1. text1: <span>foo baz</span> <span>bar</span>");-- // To separate highlights for multiple queries we'll pass them separately to the- // highlighter and differentiate highlight markers upon their application. Let's start with the customized- // field highlighter first. This utilizes the fact that match ranges passed from MatchHighlighter- // contain a reference to the original query which brought up the match.- class SeparateMarkerFieldHighlighter implements MatchHighlighter.FieldValueHighlighter {- private final String field;- private final Map<Query, String> queryClassMap;-- SeparateMarkerFieldHighlighter(String field, Map<Query, String> queryClassMap) {- this.field = field;- this.queryClassMap = queryClassMap;- }-- @Override- public boolean isApplicable(String field, boolean hasMatches) {- return Objects.equals(field, this.field) && hasMatches;- }-- @Override- public List<String> format(String field, String[] values, String contiguousValue,- List<OffsetRange> valueRanges, List<MatchHighlighter.QueryOffsetRange> matchOffsets) {- PassageSelector passageSelector = new PassageSelector();- int maxPassageWindow = 80;- int maxPassages = 3;- List<Passage> bestPassages =- passageSelector.pickBest(contiguousValue, matchOffsets, maxPassageWindow, maxPassages, valueRanges);-- // We know the offset ranges passed to us by MatchHighlighter are instances of QueryOffsetRange- // so we compute the class based on that.- Function<OffsetRange, String> queryToClass =- (range) -> queryClassMap.get(((MatchHighlighter.QueryOffsetRange) range).query);-- PassageFormatter passageFormatter = new PassageFormatter("...",- (range) -> "<span class='" + queryToClass.apply(range) + "'>",- (range) -> "</span>");-- return passageFormatter.format(contiguousValue, bestPassages, valueRanges);- }- }-- // And this is pretty much it. We now set up query classes to display, set up the highlighter...- Map<Query, String> queryClassMap = Map.of(q1, "q1", q2, "q2");- highlighter =- new MatchHighlighter(searcher, analyzer)- .appendFieldHighlighter(new SeparateMarkerFieldHighlighter(FLD_TEXT1, queryClassMap))- .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());-- // ...and run highlighting. Note the query passed to the highlighter are individual sub-clauses- // of the Boolean query used to fetch documents.- assertHighlights(toDocList(highlighter.highlight(topDocs, q1, q2)),- "0. text1: <span class='q1'>foo <span class='q2'>bar</span> baz</span>",- "1. text1: <span class='q1'>foo baz</span> <span class='q2'>bar</span>");- });+ .build(+ analyzer,+ reader -> {+ // Let's start with the two queries. The first one will be an unordered+ // query for (foo, baz) with a max gap of 1; let's use intervals for this.+ Query q1 =+ new IntervalQuery(+ FLD_TEXT1,+ Intervals.maxgaps(+ 1, Intervals.unordered(Intervals.term("foo"), Intervals.term("baz"))));++ // The second one will be a simpler term query for "bar".+ Query q2 = new TermQuery(new Term(FLD_TEXT1, "bar"));++ // Let's fetch matching documents by combining the two into a Boolean query.+ Query query =+ new BooleanQuery.Builder()+ .add(q1, BooleanClause.Occur.SHOULD)+ .add(q2, BooleanClause.Occur.SHOULD)+ .build();++ IndexSearcher searcher = new IndexSearcher(reader);+ Sort sortOrder = Sort.INDEXORDER; // So that results are consistently ordered.+ TopDocs topDocs = searcher.search(query, 10, sortOrder);++ // If we use the "regular" highlighter, the result will be slightly odd: a nested+ // highlight over "bar" within the first match. Also, you can't distinguish which of+ // the sub-queries+ // caused which highlight marker... but if it were HTML then you could give the span+ // some semi-translucent background and layered matches would be visible.+ MatchHighlighter highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(+ FieldValueHighlighters.highlighted(+ 80 * 3,+ 1,+ new PassageFormatter("...", "<span>", "</span>"),+ FLD_TEXT1::equals))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ assertHighlights(+ toDocList(highlighter.highlight(topDocs, query)),+ "0. text1: <span>foo <span>bar</span> baz</span>",+ "1. text1: <span>foo baz</span> <span>bar</span>");++ // To separate highlights for multiple queries we'll pass them separately to the+ // highlighter and differentiate highlight markers upon their application. Let's start+ // with the customized+ // field highlighter first. This utilizes the fact that match ranges passed from+ // MatchHighlighter+ // contain a reference to the original query which brought up the match.+ class SeparateMarkerFieldHighlighter+ implements MatchHighlighter.FieldValueHighlighter {+ private final String field;+ private final Map<Query, String> queryClassMap;++ SeparateMarkerFieldHighlighter(String field, Map<Query, String> queryClassMap) {+ this.field = field;+ this.queryClassMap = queryClassMap;+ }++ @Override+ public boolean isApplicable(String field, boolean hasMatches) {+ return Objects.equals(field, this.field) && hasMatches;+ }++ @Override+ public List<String> format(+ String field,+ String[] values,+ String contiguousValue,+ List<OffsetRange> valueRanges,+ List<MatchHighlighter.QueryOffsetRange> matchOffsets) {+ PassageSelector passageSelector = new PassageSelector();+ int maxPassageWindow = 80;+ int maxPassages = 3;+ List<Passage> bestPassages =+ passageSelector.pickBest(+ contiguousValue,+ matchOffsets,+ maxPassageWindow,+ maxPassages,+ valueRanges);++ // We know the offset ranges passed to us by MatchHighlighter are instances of+ // QueryOffsetRange+ // so we compute the class based on that.+ Function<OffsetRange, String> queryToClass =+ (range) ->+ queryClassMap.get(((MatchHighlighter.QueryOffsetRange) range).query);++ PassageFormatter passageFormatter =+ new PassageFormatter(+ "...",+ (range) -> "<span class='" + queryToClass.apply(range) + "'>",+ (range) -> "</span>");++ return passageFormatter.format(contiguousValue, bestPassages, valueRanges);+ }+ }++ // And this is pretty much it. We now set up query classes to display, set up the+ // highlighter...+ Map<Query, String> queryClassMap = Map.of(q1, "q1", q2, "q2");+ highlighter =+ new MatchHighlighter(searcher, analyzer)+ .appendFieldHighlighter(+ new SeparateMarkerFieldHighlighter(FLD_TEXT1, queryClassMap))+ .appendFieldHighlighter(FieldValueHighlighters.skipRemaining());++ // ...and run highlighting. Note the query passed to the highlighter are individual+ // sub-clauses+ // of the Boolean query used to fetch documents.+ assertHighlights(+ toDocList(highlighter.highlight(topDocs, q1, q2)),+ "0. text1: <span class='q1'>foo <span class='q2'>bar</span> baz</span>",+ "1. text1: <span class='q1'>foo baz</span> <span class='q2'>bar</span>");+ });}private void assertHighlights(List<List<String>> docList, String... expectedFormattedLines) {@@ -432,25 +506,30 @@ public class TestMatchHighlighter extends LuceneTestCase {for (int doc = 0; doc < docList.size(); doc++) {List<String> fields = docList.get(doc);for (int i = 0; i < fields.size(); i++) {- actualLines.add((i == 0 ? String.format(Locale.ROOT, "%2d. ", doc) : " ") + fields.get(i));+ actualLines.add(+ (i == 0 ? String.format(Locale.ROOT, "%2d. ", doc) : " ") + fields.get(i));}}if (!Arrays.equals(Stream.of(expectedFormattedLines).map(String::trim).toArray(),actualLines.stream().map(String::trim).toArray())) {- throw new AssertionError("Actual hits were:\n" +- String.join("\n", actualLines) + "\n\n but expected them to be:\n" +- String.join("\n", expectedFormattedLines));+ throw new AssertionError(+ "Actual hits were:\n"+ + String.join("\n", actualLines)+ + "\n\n but expected them to be:\n"+ + String.join("\n", expectedFormattedLines));}}private List<List<String>> toDocList(Stream<MatchHighlighter.DocHighlights> highlights) {- return highlights.map(docHighlights ->- docHighlights.fields.entrySet().stream()- .map(e -> e.getKey() + ": " + String.join(", ", e.getValue()))- .collect(Collectors.toList())- ).collect(Collectors.toList());+ return highlights+ .map(+ docHighlights ->+ docHighlights.fields.entrySet().stream()+ .map(e -> e.getKey() + ": " + String.join(", ", e.getValue()))+ .collect(Collectors.toList()))+ .collect(Collectors.toList());}private IndexableField toField(String name, String value) {diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.javaindex 08a4b8391d7..47d9c227f90 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java@@ -16,7 +16,20 @@*/package org.apache.lucene.search.matchhighlight;+import static org.hamcrest.Matchers.containsInAnyOrder;+import com.carrotsearch.randomizedtesting.RandomizedTest;+import java.io.IOException;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Set;+import java.util.TreeMap;+import java.util.function.BiFunction;+import java.util.stream.Collectors;+import java.util.stream.Stream;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CharArraySet;import org.apache.lucene.analysis.StopFilter;@@ -55,20 +68,6 @@ import org.hamcrest.Matchers;import org.junit.Before;import org.junit.Test;-import java.io.IOException;-import java.util.ArrayList;-import java.util.HashMap;-import java.util.List;-import java.util.Locale;-import java.util.Map;-import java.util.Set;-import java.util.TreeMap;-import java.util.function.BiFunction;-import java.util.stream.Collectors;-import java.util.stream.Stream;--import static org.hamcrest.Matchers.containsInAnyOrder;-public class TestMatchRegionRetriever extends LuceneTestCase {private static final String FLD_ID = IndexBuilder.FLD_ID;@@ -103,9 +102,11 @@ public class TestMatchRegionRetriever extends LuceneTestCase {TYPE_STORED_NO_POSITIONS.freeze();final int offsetGap = RandomizedTest.randomIntBetween(0, 2);- final int positionGap = RandomizedTest.randomFrom(new int[]{0, 1, 100});+ final int positionGap = RandomizedTest.randomFrom(new int[] {0, 1, 100});Analyzer whitespaceAnalyzer =- new AnalyzerWithGaps(offsetGap, positionGap,+ new AnalyzerWithGaps(+ offsetGap,+ positionGap,new Analyzer() {@Overrideprotected TokenStreamComponents createComponents(String fieldName) {@@ -116,10 +117,12 @@ public class TestMatchRegionRetriever extends LuceneTestCase {}});- SynonymMap synonymMap = TestMatchHighlighter.buildSynonymMap(new String[][] {- {"foo\u0000bar", "syn1"},- {"baz", "syn2\u0000syn3"},- });+ SynonymMap synonymMap =+ TestMatchHighlighter.buildSynonymMap(+ new String[][] {+ {"foo\u0000bar", "syn1"},+ {"baz", "syn2\u0000syn3"},+ });Analyzer synonymsAnalyzer =new Analyzer() {@@ -170,13 +173,16 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "bar foo baz").doc(field, "bar baz foo").doc(field, "bar bar bar irrelevant")- .build(analyzer, reader -> {- assertThat(highlights(reader, new TermQuery(new Term(field, "foo"))),- containsInAnyOrder(- fmt("0: (%s: '>foo< bar baz')", field),- fmt("1: (%s: 'bar >foo< baz')", field),- fmt("2: (%s: 'bar baz >foo<')", field)));- });+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, new TermQuery(new Term(field, "foo"))),+ containsInAnyOrder(+ fmt("0: (%s: '>foo< bar baz')", field),+ fmt("1: (%s: 'bar >foo< baz')", field),+ fmt("2: (%s: 'bar baz >foo<')", field)));+ });}@Test@@ -201,13 +207,15 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "foo bar baz abc").doc(field, "bar foo baz def").doc(field, "bar baz foo xyz")- .build(analyzer, reader -> {- assertThat(highlights(reader, query),- containsInAnyOrder(- fmt("0: (%s: '>foo bar baz< abc')", field),- fmt("1: (%s: 'bar >foo baz< def')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, query),+ containsInAnyOrder(+ fmt("0: (%s: '>foo bar baz< abc')", field),+ fmt("1: (%s: 'bar >foo baz< def')", field)));+ });}@Test@@ -225,71 +233,80 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "foo bar baz abc").doc(field, "bar foo baz def").doc(field, "bar baz foo xyz")- .build(analyzer, reader -> {- assertThat(highlights(reader, stdQueryParser.apply("foo baz", field)),- containsInAnyOrder(- fmt("0: (%s: '>foo< bar >baz< abc')", field),- fmt("1: (%s: 'bar >foo< >baz< def')", field),- fmt("2: (%s: 'bar >baz< >foo< xyz')", field)));-- assertThat(highlights(reader, stdQueryParser.apply("foo OR xyz", field)),- containsInAnyOrder(- fmt("0: (%s: '>foo< bar baz abc')", field),- fmt("1: (%s: 'bar >foo< baz def')", field),- fmt("2: (%s: 'bar baz >foo< >xyz<')", field)));-- assertThat(highlights(reader, stdQueryParser.apply("bas~2", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo >bar< >baz< >abc<')", field),- fmt("1: (%s: '>bar< foo >baz< def')", field),- fmt("2: (%s: '>bar< >baz< foo xyz')", field)));-- assertThat(highlights(reader, stdQueryParser.apply("\"foo bar\"", field)),- containsInAnyOrder((fmt("0: (%s: '>foo bar< baz abc')", field))));-- assertThat(highlights(reader, stdQueryParser.apply("\"foo bar\"~3", field)),- containsInAnyOrder(- fmt("0: (%s: '>foo bar< baz abc')", field),- fmt("1: (%s: '>bar foo< baz def')", field),- fmt("2: (%s: '>bar baz foo< xyz')", field)));-- assertThat(highlights(reader, stdQueryParser.apply("ba*", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo >bar< >baz< abc')", field),- fmt("1: (%s: '>bar< foo >baz< def')", field),- fmt("2: (%s: '>bar< >baz< foo xyz')", field)));-- assertThat(highlights(reader, stdQueryParser.apply("[bar TO bas]", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo >bar< baz abc')", field),- fmt("1: (%s: '>bar< foo baz def')", field),- fmt("2: (%s: '>bar< baz foo xyz')", field)));-- // Note how document '2' has 'bar' that isn't highlighted (because this- // document is excluded in the first clause).- assertThat(- highlights(reader, stdQueryParser.apply("([bar TO baz] -xyz) OR baz", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo >bar< >>baz<< abc')", field),- fmt("1: (%s: '>bar< foo >>baz<< def')", field),- fmt("2: (%s: 'bar >baz< foo xyz')", field)));-- assertThat(highlights(reader, new MatchAllDocsQuery()),- Matchers.hasSize(0));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, stdQueryParser.apply("foo baz", field)),+ containsInAnyOrder(+ fmt("0: (%s: '>foo< bar >baz< abc')", field),+ fmt("1: (%s: 'bar >foo< >baz< def')", field),+ fmt("2: (%s: 'bar >baz< >foo< xyz')", field)));++ assertThat(+ highlights(reader, stdQueryParser.apply("foo OR xyz", field)),+ containsInAnyOrder(+ fmt("0: (%s: '>foo< bar baz abc')", field),+ fmt("1: (%s: 'bar >foo< baz def')", field),+ fmt("2: (%s: 'bar baz >foo< >xyz<')", field)));++ assertThat(+ highlights(reader, stdQueryParser.apply("bas~2", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >bar< >baz< >abc<')", field),+ fmt("1: (%s: '>bar< foo >baz< def')", field),+ fmt("2: (%s: '>bar< >baz< foo xyz')", field)));++ assertThat(+ highlights(reader, stdQueryParser.apply("\"foo bar\"", field)),+ containsInAnyOrder((fmt("0: (%s: '>foo bar< baz abc')", field))));++ assertThat(+ highlights(reader, stdQueryParser.apply("\"foo bar\"~3", field)),+ containsInAnyOrder(+ fmt("0: (%s: '>foo bar< baz abc')", field),+ fmt("1: (%s: '>bar foo< baz def')", field),+ fmt("2: (%s: '>bar baz foo< xyz')", field)));++ assertThat(+ highlights(reader, stdQueryParser.apply("ba*", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >bar< >baz< abc')", field),+ fmt("1: (%s: '>bar< foo >baz< def')", field),+ fmt("2: (%s: '>bar< >baz< foo xyz')", field)));++ assertThat(+ highlights(reader, stdQueryParser.apply("[bar TO bas]", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >bar< baz abc')", field),+ fmt("1: (%s: '>bar< foo baz def')", field),+ fmt("2: (%s: '>bar< baz foo xyz')", field)));++ // Note how document '2' has 'bar' that isn't highlighted (because this+ // document is excluded in the first clause).+ assertThat(+ highlights(reader, stdQueryParser.apply("([bar TO baz] -xyz) OR baz", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >bar< >>baz<< abc')", field),+ fmt("1: (%s: '>bar< foo >>baz<< def')", field),+ fmt("2: (%s: 'bar >baz< foo xyz')", field)));++ assertThat(highlights(reader, new MatchAllDocsQuery()), Matchers.hasSize(0));+ });new IndexBuilder(this::toField).doc(field, "foo baz foo").doc(field, "bas baz foo").doc(field, "bar baz foo xyz")- .build(analyzer, reader -> {- assertThat(- highlights(reader, stdQueryParser.apply("[bar TO baz] -bar", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo >baz< foo')", field), fmt("1: (%s: '>bas< >baz< foo')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, stdQueryParser.apply("[bar TO baz] -bar", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >baz< foo')", field),+ fmt("1: (%s: '>bas< >baz< foo')", field)));+ });}@Test@@ -297,16 +314,17 @@ public class TestMatchRegionRetriever extends LuceneTestCase {String field = FLD_TEXT_POS;new IndexBuilder(this::toField).doc(field, "foo", "bar")- .build(analyzer, reader -> {- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bar")))),- containsInAnyOrder(- fmt("0: (field_text: '>foo< | >bar<')", field)- ));- });+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.unordered(Intervals.term("foo"), Intervals.term("bar")))),+ containsInAnyOrder(fmt("0: (field_text: '>foo< | >bar<')", field)));+ });}@Test@@ -317,61 +335,60 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "foo baz foo").doc(field, "bas baz foo").doc(field, "bar baz foo xyz")- .build(analyzer, reader -> {- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bas"),- Intervals.term("baz")))),- containsInAnyOrder(- fmt("1: (field_text_offs: '>bas baz foo<')", field)- ));-- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.maxgaps(1,- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bar"))))),- containsInAnyOrder(- fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)- ));-- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.containing(- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bar")),- Intervals.term("foo")))),- containsInAnyOrder(- fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)- ));-- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.containedBy(- Intervals.term("foo"),- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bar"))))),- containsInAnyOrder(- fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)- ));-- assertThat(- highlights(reader, new IntervalQuery(field,- Intervals.overlapping(- Intervals.unordered(- Intervals.term("foo"),- Intervals.term("bar")),- Intervals.term("foo")))),- containsInAnyOrder(- fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)- ));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.unordered(+ Intervals.term("foo"),+ Intervals.term("bas"),+ Intervals.term("baz")))),+ containsInAnyOrder(fmt("1: (field_text_offs: '>bas baz foo<')", field)));++ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.maxgaps(+ 1,+ Intervals.unordered(Intervals.term("foo"), Intervals.term("bar"))))),+ containsInAnyOrder(fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)));++ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.containing(+ Intervals.unordered(Intervals.term("foo"), Intervals.term("bar")),+ Intervals.term("foo")))),+ containsInAnyOrder(fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)));++ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.containedBy(+ Intervals.term("foo"),+ Intervals.unordered(Intervals.term("foo"), Intervals.term("bar"))))),+ containsInAnyOrder(fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)));++ assertThat(+ highlights(+ reader,+ new IntervalQuery(+ field,+ Intervals.overlapping(+ Intervals.unordered(Intervals.term("foo"), Intervals.term("bar")),+ Intervals.term("foo")))),+ containsInAnyOrder(fmt("2: (field_text_offs: '>bar baz foo< xyz')", field)));+ });}@Test@@ -379,8 +396,11 @@ public class TestMatchRegionRetriever extends LuceneTestCase {testDegenerateIntervals(FLD_TEXT_POS);}- @Test @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-9634: " +- "Highlighting of degenerate spans on fields with offsets doesn't work properly")+ @Test+ @AwaitsFix(+ bugUrl =+ "https://issues.apache.org/jira/browse/LUCENE-9634: "+ + "Highlighting of degenerate spans on fields with offsets doesn't work properly")public void testDegenerateIntervalsWithOffsets() throws IOException {testDegenerateIntervals(FLD_TEXT_POS_OFFS);}@@ -388,25 +408,21 @@ public class TestMatchRegionRetriever extends LuceneTestCase {public void testDegenerateIntervals(String field) throws IOException {new IndexBuilder(this::toField).doc(field, fmt("foo %s bar", STOPWORD1))- .build(analyzer, reader -> {- assertThat(- highlights(reader, new IntervalQuery(- field,- Intervals.extend(- Intervals.term("bar"), 1, 3))),- containsInAnyOrder(- fmt("0: (%s: 'foo %s >bar<')", field, STOPWORD1)- ));-- assertThat(- highlights(reader, new IntervalQuery(- field,- Intervals.extend(- Intervals.term("bar"), 5, 100))),- containsInAnyOrder(- fmt("0: (%s: '>foo %s bar<')", field, STOPWORD1)- ));- });+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(+ reader,+ new IntervalQuery(field, Intervals.extend(Intervals.term("bar"), 1, 3))),+ containsInAnyOrder(fmt("0: (%s: 'foo %s >bar<')", field, STOPWORD1)));++ assertThat(+ highlights(+ reader,+ new IntervalQuery(field, Intervals.extend(Intervals.term("bar"), 5, 100))),+ containsInAnyOrder(fmt("0: (%s: '>foo %s bar<')", field, STOPWORD1)));+ });}@Test@@ -424,45 +440,49 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "foo bar", "baz abc", "bad baz").doc(field, "bar foo", "baz def").doc(field, "bar baz", "foo xyz")- .build(analyzer, reader -> {- assertThat(highlights(reader, stdQueryParser.apply("baz", field)),- containsInAnyOrder(- fmt("0: (%s: '>baz< abc | bad >baz<')", field),- fmt("1: (%s: '>baz< def')", field),- fmt("2: (%s: 'bar >baz<')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, stdQueryParser.apply("baz", field)),+ containsInAnyOrder(+ fmt("0: (%s: '>baz< abc | bad >baz<')", field),+ fmt("1: (%s: '>baz< def')", field),+ fmt("2: (%s: 'bar >baz<')", field)));+ });}@Testpublic void testMultiFieldHighlights() throws IOException {for (String[] fieldPairs :- new String[][]{- {FLD_TEXT_POS_OFFS1, FLD_TEXT_POS_OFFS2},- {FLD_TEXT_POS, FLD_TEXT_POS_OFFS2},- {FLD_TEXT_POS_OFFS1, FLD_TEXT_POS}+ new String[][] {+ {FLD_TEXT_POS_OFFS1, FLD_TEXT_POS_OFFS2},+ {FLD_TEXT_POS, FLD_TEXT_POS_OFFS2},+ {FLD_TEXT_POS_OFFS1, FLD_TEXT_POS}}) {String field1 = fieldPairs[0];String field2 = fieldPairs[1];new IndexBuilder(this::toField)- .doc(fields -> {- fields.add(field1, "foo bar", "baz abc");- fields.add(field2, "foo baz", "loo bar");- })- .build(analyzer, reader -> {- String ordered =- Stream.of(fmt("(%s: '>baz< abc')", field1), fmt("(%s: 'loo >bar<')", field2))- .sorted()- .collect(Collectors.joining(""));-- assertThat(- highlights(- reader,- stdQueryParser.apply(field1 + ":baz" + " OR " + field2 + ":bar", field1)),- containsInAnyOrder(fmt("0: %s", ordered)));- }- );+ .doc(+ fields -> {+ fields.add(field1, "foo bar", "baz abc");+ fields.add(field2, "foo baz", "loo bar");+ })+ .build(+ analyzer,+ reader -> {+ String ordered =+ Stream.of(fmt("(%s: '>baz< abc')", field1), fmt("(%s: 'loo >bar<')", field2))+ .sorted()+ .collect(Collectors.joining(""));++ assertThat(+ highlights(+ reader,+ stdQueryParser.apply(field1 + ":baz" + " OR " + field2 + ":bar", field1)),+ containsInAnyOrder(fmt("0: %s", ordered)));+ });}}@@ -476,29 +496,32 @@ public class TestMatchRegionRetriever extends LuceneTestCase {String field2 = FLD_TEXT_POS_OFFS2;new IndexBuilder(this::toField)- .doc(fields -> {- fields.add(field1, "0100");- fields.add(field2, "loo bar");- })- .doc(fields -> {- fields.add(field1, "0200");- fields.add(field2, "foo bar");- })- .build(analyzer, reader -> {- String expected = fmt("0: (%s: '>0100<')(%s: 'loo >bar<')", field1, field2);- assertThat(- highlights(- reader,- stdQueryParser.apply(fmt("+%s:01* OR %s:bar", field1, field2), field1)),- containsInAnyOrder(expected));-- assertThat(- highlights(- reader,- stdQueryParser.apply(fmt("+%s:01* AND %s:bar", field1, field2), field1)),- containsInAnyOrder(expected));- }- );+ .doc(+ fields -> {+ fields.add(field1, "0100");+ fields.add(field2, "loo bar");+ })+ .doc(+ fields -> {+ fields.add(field1, "0200");+ fields.add(field2, "foo bar");+ })+ .build(+ analyzer,+ reader -> {+ String expected = fmt("0: (%s: '>0100<')(%s: 'loo >bar<')", field1, field2);+ assertThat(+ highlights(+ reader,+ stdQueryParser.apply(fmt("+%s:01* OR %s:bar", field1, field2), field1)),+ containsInAnyOrder(expected));++ assertThat(+ highlights(+ reader,+ stdQueryParser.apply(fmt("+%s:01* AND %s:bar", field1, field2), field1)),+ containsInAnyOrder(expected));+ });}@Test@@ -514,27 +537,28 @@ public class TestMatchRegionRetriever extends LuceneTestCase {public void checkNestedQueryHits(String field) throws IOException {new IndexBuilder(this::toField).doc(field, "foo bar baz abc")- .build(analyzer, reader -> {- assertThat(- highlights(- reader,- new BooleanQuery.Builder()- .add(new PhraseQuery(1, field, "foo", "baz"), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term(field, "bar")), BooleanClause.Occur.SHOULD)- .build()),- containsInAnyOrder(fmt("0: (%s: '>foo >bar< baz< abc')", field)));-- assertThat(- highlights(- reader,- new BooleanQuery.Builder()- .add(new PhraseQuery(1, field, "foo", "baz"), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term(field, "bar")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term(field, "baz")), BooleanClause.Occur.SHOULD)- .build()),- containsInAnyOrder(fmt("0: (%s: '>foo >bar< >baz<< abc')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(+ reader,+ new BooleanQuery.Builder()+ .add(new PhraseQuery(1, field, "foo", "baz"), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term(field, "bar")), BooleanClause.Occur.SHOULD)+ .build()),+ containsInAnyOrder(fmt("0: (%s: '>foo >bar< baz< abc')", field)));++ assertThat(+ highlights(+ reader,+ new BooleanQuery.Builder()+ .add(new PhraseQuery(1, field, "foo", "baz"), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term(field, "bar")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term(field, "baz")), BooleanClause.Occur.SHOULD)+ .build()),+ containsInAnyOrder(fmt("0: (%s: '>foo >bar< >baz<< abc')", field)));+ });}@Test@@ -553,28 +577,31 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "bar foo baz").doc(field, "bar baz foo").doc(field, "bar bar bar irrelevant")- .build(analyzer, reader -> {- assertThat(highlights(reader, new TermQuery(new Term(field, "syn1"))),- containsInAnyOrder(fmt("0: (%s: '>foo bar< baz')", field)));-- // [syn2 syn3] = baz- // so both these queries highlight baz.- assertThat(highlights(reader, new TermQuery(new Term(field, "syn3"))),- containsInAnyOrder(- fmt("0: (%s: 'foo bar >baz<')", field),- fmt("1: (%s: 'bar foo >baz<')", field),- fmt("2: (%s: 'bar >baz< foo')", field)));- assertThat(- highlights(reader, stdQueryParser.apply(field + ":\"syn2 syn3\"", field)),- containsInAnyOrder(- fmt("0: (%s: 'foo bar >baz<')", field),- fmt("1: (%s: 'bar foo >baz<')", field),- fmt("2: (%s: 'bar >baz< foo')", field)));- assertThat(- highlights(reader, stdQueryParser.apply(field + ":\"foo syn2 syn3\"", field)),- containsInAnyOrder(fmt("1: (%s: 'bar >foo baz<')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, new TermQuery(new Term(field, "syn1"))),+ containsInAnyOrder(fmt("0: (%s: '>foo bar< baz')", field)));++ // [syn2 syn3] = baz+ // so both these queries highlight baz.+ assertThat(+ highlights(reader, new TermQuery(new Term(field, "syn3"))),+ containsInAnyOrder(+ fmt("0: (%s: 'foo bar >baz<')", field),+ fmt("1: (%s: 'bar foo >baz<')", field),+ fmt("2: (%s: 'bar >baz< foo')", field)));+ assertThat(+ highlights(reader, stdQueryParser.apply(field + ":\"syn2 syn3\"", field)),+ containsInAnyOrder(+ fmt("0: (%s: 'foo bar >baz<')", field),+ fmt("1: (%s: 'bar foo >baz<')", field),+ fmt("2: (%s: 'bar >baz< foo')", field)));+ assertThat(+ highlights(reader, stdQueryParser.apply(field + ":\"foo syn2 syn3\"", field)),+ containsInAnyOrder(fmt("1: (%s: 'bar >foo baz<')", field)));+ });}@Test@@ -593,56 +620,58 @@ public class TestMatchRegionRetriever extends LuceneTestCase {.doc(field, "bar foo baz").doc(field, "bar baz foo").doc(field, "bar bar bar irrelevant")- .build(analyzer, reader -> {- assertThat(- highlights(- reader,- SpanNearQuery.newOrderedNearQuery(field)- .addClause(new SpanTermQuery(new Term(field, "bar")))- .addClause(new SpanTermQuery(new Term(field, "foo")))- .build()),- containsInAnyOrder(fmt("1: (%s: '>bar foo< baz')", field)));-- assertThat(- highlights(- reader,- SpanNearQuery.newOrderedNearQuery(field)- .addClause(new SpanTermQuery(new Term(field, "bar")))- .addGap(1)- .addClause(new SpanTermQuery(new Term(field, "foo")))- .build()),- containsInAnyOrder(fmt("2: (%s: '>bar baz foo<')", field)));-- assertThat(- highlights(- reader,- SpanNearQuery.newUnorderedNearQuery(field)- .addClause(new SpanTermQuery(new Term(field, "foo")))- .addClause(new SpanTermQuery(new Term(field, "bar")))- .build()),- containsInAnyOrder(- fmt("0: (%s: '>foo bar< baz')", field), fmt("1: (%s: '>bar foo< baz')", field)));-- assertThat(- highlights(- reader,- SpanNearQuery.newUnorderedNearQuery(field)- .addClause(new SpanTermQuery(new Term(field, "foo")))- .addClause(new SpanTermQuery(new Term(field, "bar")))- .setSlop(1)- .build()),- containsInAnyOrder(- fmt("0: (%s: '>foo bar< baz')", field),- fmt("1: (%s: '>bar foo< baz')", field),- fmt("2: (%s: '>bar baz foo<')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(+ reader,+ SpanNearQuery.newOrderedNearQuery(field)+ .addClause(new SpanTermQuery(new Term(field, "bar")))+ .addClause(new SpanTermQuery(new Term(field, "foo")))+ .build()),+ containsInAnyOrder(fmt("1: (%s: '>bar foo< baz')", field)));++ assertThat(+ highlights(+ reader,+ SpanNearQuery.newOrderedNearQuery(field)+ .addClause(new SpanTermQuery(new Term(field, "bar")))+ .addGap(1)+ .addClause(new SpanTermQuery(new Term(field, "foo")))+ .build()),+ containsInAnyOrder(fmt("2: (%s: '>bar baz foo<')", field)));++ assertThat(+ highlights(+ reader,+ SpanNearQuery.newUnorderedNearQuery(field)+ .addClause(new SpanTermQuery(new Term(field, "foo")))+ .addClause(new SpanTermQuery(new Term(field, "bar")))+ .build()),+ containsInAnyOrder(+ fmt("0: (%s: '>foo bar< baz')", field),+ fmt("1: (%s: '>bar foo< baz')", field)));++ assertThat(+ highlights(+ reader,+ SpanNearQuery.newUnorderedNearQuery(field)+ .addClause(new SpanTermQuery(new Term(field, "foo")))+ .addClause(new SpanTermQuery(new Term(field, "bar")))+ .setSlop(1)+ .build()),+ containsInAnyOrder(+ fmt("0: (%s: '>foo bar< baz')", field),+ fmt("1: (%s: '>bar foo< baz')", field),+ fmt("2: (%s: '>bar baz foo<')", field)));+ });}/**- * This test runs a term query against a field with no stored- * positions or offsets. This test checks the {@link OffsetsFromValues}- * strategy that returns highlights over entire indexed values.+ * This test runs a term query against a field with no stored positions or offsets. This test+ * checks the {@link OffsetsFromValues} strategy that returns highlights over entire indexed+ * values.*/@Testpublic void testTextFieldNoPositionsOffsetFromValues() throws Exception {@@ -651,65 +680,63 @@ public class TestMatchRegionRetriever extends LuceneTestCase {new IndexBuilder(this::toField).doc(FLD_TEXT_NOPOS, "foo bar").doc(FLD_TEXT_NOPOS, "foo bar", "baz baz")- .build(analyzer, reader -> {- OffsetsRetrievalStrategySupplier defaults = MatchRegionRetriever- .computeOffsetRetrievalStrategies(reader, analyzer);- OffsetsRetrievalStrategySupplier customSuppliers = (fld) -> {- if (fld.equals(field)) {- return new OffsetsFromValues(field, analyzer);- } else {- return defaults.apply(field);- }- };-- assertThat(- highlights(- customSuppliers,- reader,- new TermQuery(new Term(field, "bar"))),- containsInAnyOrder(- fmt("0: (%s: '>foo bar<')", field),- fmt("1: (%s: '>foo bar< | >baz baz<')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ OffsetsRetrievalStrategySupplier defaults =+ MatchRegionRetriever.computeOffsetRetrievalStrategies(reader, analyzer);+ OffsetsRetrievalStrategySupplier customSuppliers =+ (fld) -> {+ if (fld.equals(field)) {+ return new OffsetsFromValues(field, analyzer);+ } else {+ return defaults.apply(field);+ }+ };++ assertThat(+ highlights(customSuppliers, reader, new TermQuery(new Term(field, "bar"))),+ containsInAnyOrder(+ fmt("0: (%s: '>foo bar<')", field),+ fmt("1: (%s: '>foo bar< | >baz baz<')", field)));+ });}/**- * This test runs a term query against a field with no stored- * positions or offsets.- * <p>- * Such field structure is often useful for multivalued "keyword-like"- * fields.+ * This test runs a term query against a field with no stored positions or offsets.+ *+ * <p>Such field structure is often useful for multivalued "keyword-like" fields.*/@Testpublic void testTextFieldNoPositionsOffsetsFromTokens() throws Exception {String field = FLD_TEXT_NOPOS;new IndexBuilder(this::toField)- .doc(fields -> {- fields.add(FLD_TEXT_NOPOS, "foo bar");- fields.add(FLD_TEXT_POS, "bar bar");- })+ .doc(+ fields -> {+ fields.add(FLD_TEXT_NOPOS, "foo bar");+ fields.add(FLD_TEXT_POS, "bar bar");+ }).doc(FLD_TEXT_NOPOS, "foo bar", "baz bar")- .build(analyzer, reader -> {- assertThat(- highlights(- reader,- new TermQuery(new Term(field, "bar"))),- containsInAnyOrder(- fmt("0: (%s: 'foo >bar<')", field),- fmt("1: (%s: 'foo >bar< | baz >bar<')", field)));- }- );+ .build(+ analyzer,+ reader -> {+ assertThat(+ highlights(reader, new TermQuery(new Term(field, "bar"))),+ containsInAnyOrder(+ fmt("0: (%s: 'foo >bar<')", field),+ fmt("1: (%s: 'foo >bar< | baz >bar<')", field)));+ });}private List<String> highlights(IndexReader reader, Query query) throws IOException {- return highlights(MatchRegionRetriever.computeOffsetRetrievalStrategies(reader, analyzer),- reader, query);+ return highlights(+ MatchRegionRetriever.computeOffsetRetrievalStrategies(reader, analyzer), reader, query);}- private List<String> highlights(OffsetsRetrievalStrategySupplier offsetsStrategySupplier,- IndexReader reader, Query query) throws IOException {+ private List<String> highlights(+ OffsetsRetrievalStrategySupplier offsetsStrategySupplier, IndexReader reader, Query query)+ throws IOException {IndexSearcher searcher = new IndexSearcher(reader);int maxDocs = 1000;@@ -740,7 +767,8 @@ public class TestMatchRegionRetriever extends LuceneTestCase {}};- MatchRegionRetriever highlighter = new MatchRegionRetriever(searcher, rewrittenQuery, offsetsStrategySupplier);+ MatchRegionRetriever highlighter =+ new MatchRegionRetriever(searcher, rewrittenQuery, offsetsStrategySupplier);highlighter.highlightDocuments(topDocs, highlightCollector);return highlights;diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.javaindex e1472bb937a..23a17a13c30 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java@@ -16,16 +16,15 @@*/package org.apache.lucene.search.matchhighlight;-import org.apache.lucene.util.LuceneTestCase;-import org.hamcrest.Matchers;-import org.junit.Test;+import static com.carrotsearch.randomizedtesting.RandomizedTest.*;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Objects;--import static com.carrotsearch.randomizedtesting.RandomizedTest.*;+import org.apache.lucene.util.LuceneTestCase;+import org.hamcrest.Matchers;+import org.junit.Test;public class TestPassageSelector extends LuceneTestCase {@Test@@ -74,7 +73,7 @@ public class TestPassageSelector extends LuceneTestCase {15,2,new OffsetRange[0],- new OffsetRange[]{new OffsetRange(0, 4), new OffsetRange(4, 9)});+ new OffsetRange[] {new OffsetRange(0, 4), new OffsetRange(4, 9)});}@Testdiff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.javaindex cacb9cbb9a2..0c4004f3ddb 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestCustomSeparatorBreakIterator.java@@ -16,23 +16,31 @@*/package org.apache.lucene.search.uhighlight;-import java.text.BreakIterator;-import java.util.Locale;+import static org.apache.lucene.search.uhighlight.TestWholeBreakIterator.assertSameBreaks;+import static org.hamcrest.CoreMatchers.equalTo;import com.carrotsearch.randomizedtesting.generators.RandomPicks;+import java.text.BreakIterator;+import java.util.Locale;import org.apache.lucene.util.LuceneTestCase;-import static org.apache.lucene.search.uhighlight.TestWholeBreakIterator.assertSameBreaks;-import static org.hamcrest.CoreMatchers.equalTo;-public class TestCustomSeparatorBreakIterator extends LuceneTestCase {- private static final Character[] SEPARATORS = new Character[]{' ', '\u0000', 8233};+ private static final Character[] SEPARATORS = new Character[] {' ', '\u0000', 8233};public void testBreakOnCustomSeparator() throws Exception {Character separator = randomSeparator();BreakIterator bi = new CustomSeparatorBreakIterator(separator);- String source = "this" + separator + "is" + separator + "the" + separator + "first" + separator + "sentence";+ String source =+ "this"+ + separator+ + "is"+ + separator+ + "the"+ + separator+ + "first"+ + separator+ + "sentence";bi.setText(source);assertThat(bi.current(), equalTo(0));assertThat(bi.first(), equalTo(0));@@ -57,12 +65,17 @@ public class TestCustomSeparatorBreakIterator extends LuceneTestCase {assertThat(bi.previous(), equalTo(BreakIterator.DONE));assertThat(bi.current(), equalTo(0));- assertThat(source.substring(0, bi.following(9)), equalTo("this" + separator + "is" + separator + "the" + separator));+ assertThat(+ source.substring(0, bi.following(9)),+ equalTo("this" + separator + "is" + separator + "the" + separator));- assertThat(source.substring(0, bi.preceding(9)), equalTo("this" + separator + "is" + separator));+ assertThat(+ source.substring(0, bi.preceding(9)), equalTo("this" + separator + "is" + separator));assertThat(bi.first(), equalTo(0));- assertThat(source.substring(0, bi.next(3)), equalTo("this" + separator + "is" + separator + "the" + separator));+ assertThat(+ source.substring(0, bi.next(3)),+ equalTo("this" + separator + "is" + separator + "the" + separator));}public void testSingleSentences() throws Exception {diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestDefaultPassageFormatter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestDefaultPassageFormatter.javaindex 8389dbe0c97..15fb96eb26a 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestDefaultPassageFormatter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestDefaultPassageFormatter.java@@ -21,31 +21,35 @@ import org.apache.lucene.util.LuceneTestCase;public class TestDefaultPassageFormatter extends LuceneTestCase {public void testBasic() throws Exception {- String text = "Test customization & <div class=\"xy\">"escaping"</div> of this very formatter. Unrelated part. It's not very N/A!";+ String text =+ "Test customization & <div class=\"xy\">"escaping"</div> of this very formatter. Unrelated part. It's not very N/A!";// fabricate passages with matches to formatPassage[] passages = new Passage[2];passages[0] = new Passage();passages[0].setStartOffset(0);- passages[0].setEndOffset(text.indexOf(".")+1);- passages[0].addMatch(text.indexOf("very"), text.indexOf("very")+4, null, 2);+ passages[0].setEndOffset(text.indexOf(".") + 1);+ passages[0].addMatch(text.indexOf("very"), text.indexOf("very") + 4, null, 2);passages[1] = new Passage();- passages[1].setStartOffset(text.indexOf(".", passages[0].getEndOffset()+1) + 2);+ passages[1].setStartOffset(text.indexOf(".", passages[0].getEndOffset() + 1) + 2);passages[1].setEndOffset(text.length());passages[1].addMatch(text.indexOf("very", passages[0].getEndOffset()),- text.indexOf("very", passages[0].getEndOffset())+4, null, 2);+ text.indexOf("very", passages[0].getEndOffset()) + 4,+ null,+ 2);// test defaultDefaultPassageFormatter formatter = new DefaultPassageFormatter();assertEquals(- "Test customization & <div class=\"xy\">"escaping"</div> of this <b>very</b> formatter." +- "... It's not <b>very</b> N/A!", formatter.format(passages, text));+ "Test customization & <div class=\"xy\">"escaping"</div> of this <b>very</b> formatter."+ + "... It's not <b>very</b> N/A!",+ formatter.format(passages, text));// test customization and encodingformatter = new DefaultPassageFormatter("<u>", "</u>", "\u2026 ", true);assertEquals(- "Test customization & <div class="xy">&quot;escaping&quot;" +- "</div> of this <u>very</u> formatter.\u2026 It's not <u>very</u> N/A!",+ "Test customization & <div class="xy">&quot;escaping&quot;"+ + "</div> of this <u>very</u> formatter.\u2026 It's not <u>very</u> N/A!",formatter.format(passages, text));}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestLengthGoalBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestLengthGoalBreakIterator.javaindex 0ee73a8c1a1..85d467d3867 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestLengthGoalBreakIterator.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestLengthGoalBreakIterator.java@@ -19,7 +19,6 @@ package org.apache.lucene.search.uhighlight;import java.io.IOException;import java.text.BreakIterator;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -31,11 +30,13 @@ public class TestLengthGoalBreakIterator extends LuceneTestCase {private static final String FIELD = "body";private static final float[] ALIGNS = {0.f, 0.5f, 1.f};- // We test LengthGoalBreakIterator as it is used by the UnifiedHighlighter instead of directly, because it is- // not a general purpose BreakIterator. A unit test of it directly wouldn't give as much confidence.+ // We test LengthGoalBreakIterator as it is used by the UnifiedHighlighter instead of directly,+ // because it is+ // not a general purpose BreakIterator. A unit test of it directly wouldn't give as much+ // confidence.private final Analyzer analyzer =- new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercase// We do a '.' BreakIterator and test varying the length goal.// 0 1@@ -51,14 +52,19 @@ public class TestLengthGoalBreakIterator extends LuceneTestCase {for (float alignment : valid_aligns) {LengthGoalBreakIterator.createClosestToLength(baseBI, 50, alignment);}- float[] invalid_aligns = {-0.01f, -1.f, 1.5f, Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY};+ float[] invalid_aligns = {+ -0.01f, -1.f, 1.5f, Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY+ };for (float alignment : invalid_aligns) {- expectThrows(IllegalArgumentException.class, () -> {- LengthGoalBreakIterator.createClosestToLength(baseBI, 50, alignment);- });+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ LengthGoalBreakIterator.createClosestToLength(baseBI, 50, alignment);+ });}// test backwards compatibility constructors- String backwardCompString = LengthGoalBreakIterator.createClosestToLength(baseBI, 50).toString();+ String backwardCompString =+ LengthGoalBreakIterator.createClosestToLength(baseBI, 50).toString();assertTrue(backwardCompString, backwardCompString.contains("fragAlign=0.0"));backwardCompString = LengthGoalBreakIterator.createMinLength(baseBI, 50).toString();assertTrue(backwardCompString, backwardCompString.contains("fragAlign=0.0"));@@ -69,131 +75,177 @@ public class TestLengthGoalBreakIterator extends LuceneTestCase {// at first word:Query query = query("aa");- assertEquals("almost two sent A",- "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 7, 0.f));- assertEquals("almost two sent B",- "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 15, 0.5f));- assertEquals("almost two sent C",- "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 64, 1.f));- assertEquals("barely two sent A",- "<b>Aa</b> bb. Cc dd.", highlightClosestToLen(CONTENT, query, 8, 0.f));- assertEquals("barely two sent B",- "<b>Aa</b> bb. Cc dd.", highlightClosestToLen(CONTENT, query, 16, 0.5f));- assertEquals("long goal A",- "<b>Aa</b> bb. Cc dd. Ee ff", highlightClosestToLen(CONTENT, query, 14 + random().nextInt(20), 0.f));- assertEquals("long goal B",- "<b>Aa</b> bb. Cc dd. Ee ff", highlightClosestToLen(CONTENT, query, 28 + random().nextInt(20), 0.5f));+ assertEquals(+ "almost two sent A", "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 7, 0.f));+ assertEquals(+ "almost two sent B", "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 15, 0.5f));+ assertEquals(+ "almost two sent C", "<b>Aa</b> bb.", highlightClosestToLen(CONTENT, query, 64, 1.f));+ assertEquals(+ "barely two sent A", "<b>Aa</b> bb. Cc dd.", highlightClosestToLen(CONTENT, query, 8, 0.f));+ assertEquals(+ "barely two sent B",+ "<b>Aa</b> bb. Cc dd.",+ highlightClosestToLen(CONTENT, query, 16, 0.5f));+ assertEquals(+ "long goal A",+ "<b>Aa</b> bb. Cc dd. Ee ff",+ highlightClosestToLen(CONTENT, query, 14 + random().nextInt(20), 0.f));+ assertEquals(+ "long goal B",+ "<b>Aa</b> bb. Cc dd. Ee ff",+ highlightClosestToLen(CONTENT, query, 28 + random().nextInt(20), 0.5f));// at some word not at start of passagequery = query("dd");for (float align : ALIGNS) {- // alignment is not meaningful if fragsize is shorter than or closer to match-fragment boundaries- assertEquals("short goal " + align,- " Cc <b>dd</b>.", highlightClosestToLen(CONTENT, query, random().nextInt(4), align));+ // alignment is not meaningful if fragsize is shorter than or closer to match-fragment+ // boundaries+ assertEquals(+ "short goal " + align,+ " Cc <b>dd</b>.",+ highlightClosestToLen(CONTENT, query, random().nextInt(4), align));}// preceding/following inclusion by alignment parameter- assertEquals("barely two sent A",- " Cc <b>dd</b>. Ee ff", highlightClosestToLen(CONTENT, query, 11, 0.f));- assertEquals("barely two sent B",- " Cc <b>dd</b>. Ee ff", highlightClosestToLen(CONTENT, query, 11, 0.5f));- assertEquals("barely two sent C",- "Aa bb. Cc <b>dd</b>.", highlightClosestToLen(CONTENT, query, 11, 1.f));- assertEquals("long goal A",- " Cc <b>dd</b>. Ee ff", highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 0.f));- assertEquals("long goal B",- "Aa bb. Cc <b>dd</b>. Ee ff", highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 0.5f));- assertEquals("long goal C",- "Aa bb. Cc <b>dd</b>.", highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 1.f));+ assertEquals(+ "barely two sent A",+ " Cc <b>dd</b>. Ee ff",+ highlightClosestToLen(CONTENT, query, 11, 0.f));+ assertEquals(+ "barely two sent B",+ " Cc <b>dd</b>. Ee ff",+ highlightClosestToLen(CONTENT, query, 11, 0.5f));+ assertEquals(+ "barely two sent C",+ "Aa bb. Cc <b>dd</b>.",+ highlightClosestToLen(CONTENT, query, 11, 1.f));+ assertEquals(+ "long goal A",+ " Cc <b>dd</b>. Ee ff",+ highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 0.f));+ assertEquals(+ "long goal B",+ "Aa bb. Cc <b>dd</b>. Ee ff",+ highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 0.5f));+ assertEquals(+ "long goal C",+ "Aa bb. Cc <b>dd</b>.",+ highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), 1.f));query = query("ddxyzee");- assertEquals("test fragment search from the middle of the match; almost including",- "<b>ddxyzee</b> ", highlightClosestToLen(CONTENT3, query, 7, 0.5f, 1, ' '));- assertEquals("test fragment search from the middle of the match; barely including",- "bbcc <b>ddxyzee</b> ffgg ", highlightClosestToLen(CONTENT3, query, 14, 0.5f, 1, ' '));+ assertEquals(+ "test fragment search from the middle of the match; almost including",+ "<b>ddxyzee</b> ",+ highlightClosestToLen(CONTENT3, query, 7, 0.5f, 1, ' '));+ assertEquals(+ "test fragment search from the middle of the match; barely including",+ "bbcc <b>ddxyzee</b> ffgg ",+ highlightClosestToLen(CONTENT3, query, 14, 0.5f, 1, ' '));}public void testMinLen() throws IOException {// minLen mode is simpler than targetLen... just test a few casesQuery query = query("dd");- assertEquals("almost two sent A",- " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 0, 0.f));- assertEquals("almost two sent B",- " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 1, 0.5f));- assertEquals("almost two sent C",- " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 5, 1.f));-- assertEquals("barely two sent A",- " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 1, 0.f));- assertEquals("barely two sent B",- " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 2, 0.5f));- assertEquals("barely two sent C",- "Aa bb. Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 7, 1.f));- assertEquals("barely two sent D/a",- " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 2, 0.55f));- assertEquals("barely two sent D/b",- " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 3, 0.55f));- assertEquals("barely two sent E/a",- " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 10, 0.5f));- assertEquals("barely two sent E/b",- "Aa bb. Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 10, 0.7f));- assertEquals("barely two sent E/c",- "Aa bb. Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 9, 0.9f));+ assertEquals("almost two sent A", " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 0, 0.f));+ assertEquals("almost two sent B", " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 1, 0.5f));+ assertEquals("almost two sent C", " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 5, 1.f));++ assertEquals(+ "barely two sent A", " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 1, 0.f));+ assertEquals(+ "barely two sent B", " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 2, 0.5f));+ assertEquals(+ "barely two sent C", "Aa bb. Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 7, 1.f));+ assertEquals(+ "barely two sent D/a", " Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 2, 0.55f));+ assertEquals(+ "barely two sent D/b", " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 3, 0.55f));+ assertEquals(+ "barely two sent E/a", " Cc <b>dd</b>. Ee ff", highlightMinLen(CONTENT, query, 10, 0.5f));+ assertEquals(+ "barely two sent E/b",+ "Aa bb. Cc <b>dd</b>. Ee ff",+ highlightMinLen(CONTENT, query, 10, 0.7f));+ assertEquals(+ "barely two sent E/c", "Aa bb. Cc <b>dd</b>.", highlightMinLen(CONTENT, query, 9, 0.9f));query = query("ddxyzee");- assertEquals("test fragment search from the middle of the match; almost including",- "<b>ddxyzee</b> ", highlightMinLen(CONTENT3, query, 7, 0.5f, ' '));- assertEquals("test fragment search from the middle of the match; barely including",- "bbcc <b>ddxyzee</b> ffgg ", highlightMinLen(CONTENT3, query, 8, 0.5f, ' '));+ assertEquals(+ "test fragment search from the middle of the match; almost including",+ "<b>ddxyzee</b> ",+ highlightMinLen(CONTENT3, query, 7, 0.5f, ' '));+ assertEquals(+ "test fragment search from the middle of the match; barely including",+ "bbcc <b>ddxyzee</b> ffgg ",+ highlightMinLen(CONTENT3, query, 8, 0.5f, ' '));}public void testMinLenPrecision() throws IOException {Query query = query("x");- assertEquals("test absolute minimal length",- "<b>X</b> ", highlightMinLen(CONTENT2, query, 1, 0.5f, ' '));- assertEquals("test slightly above minimal length",- "dd <b>X</b> Ee ", highlightMinLen(CONTENT2, query, 4, 0.5f, ' '));+ assertEquals(+ "test absolute minimal length",+ "<b>X</b> ",+ highlightMinLen(CONTENT2, query, 1, 0.5f, ' '));+ assertEquals(+ "test slightly above minimal length",+ "dd <b>X</b> Ee ",+ highlightMinLen(CONTENT2, query, 4, 0.5f, ' '));}public void testDefaultSummaryTargetLen() throws IOException {Query query = query("zz");for (float align : ALIGNS) { // alignment is not used for creating default-summary- assertEquals("Aa bb.",- highlightClosestToLen(CONTENT, query, 6 + random().nextInt(4), align));- assertEquals("Aa bb. Cc dd.",- highlightClosestToLen(CONTENT, query, 12 + random().nextInt(4), align));- assertEquals("Aa bb. Cc dd. Ee ff",+ assertEquals("Aa bb.", highlightClosestToLen(CONTENT, query, 6 + random().nextInt(4), align));+ assertEquals(+ "Aa bb. Cc dd.", highlightClosestToLen(CONTENT, query, 12 + random().nextInt(4), align));+ assertEquals(+ "Aa bb. Cc dd. Ee ff",highlightClosestToLen(CONTENT, query, 17 + random().nextInt(20), align));}- assertEquals("Aa bb. Cc dd.",- highlightClosestToLen(CONTENT, query, 6 + random().nextInt(4), 0.f, 2));+ assertEquals(+ "Aa bb. Cc dd.", highlightClosestToLen(CONTENT, query, 6 + random().nextInt(4), 0.f, 2));}private Query query(String qStr) {return new QueryBuilder(analyzer).createBooleanQuery(FIELD, qStr);}- private String highlightClosestToLen(String content, Query query, int lengthGoal, float fragAlign) throws IOException {+ private String highlightClosestToLen(String content, Query query, int lengthGoal, float fragAlign)+ throws IOException {return highlightClosestToLen(content, query, lengthGoal, fragAlign, 1);}- private String highlightClosestToLen(String content, Query query, int lengthGoal, float fragAlign, int maxPassages) throws IOException {+ private String highlightClosestToLen(+ String content, Query query, int lengthGoal, float fragAlign, int maxPassages)+ throws IOException {return highlightClosestToLen(content, query, lengthGoal, fragAlign, maxPassages, '.');}- private String highlightClosestToLen(String content, Query query, int lengthGoal, float fragAlign, int maxPassages, char separator) throws IOException {+ private String highlightClosestToLen(+ String content, Query query, int lengthGoal, float fragAlign, int maxPassages, char separator)+ throws IOException {UnifiedHighlighter highlighter = new UnifiedHighlighter(null, analyzer);- highlighter.setBreakIterator(() -> LengthGoalBreakIterator.createClosestToLength(new CustomSeparatorBreakIterator(separator), lengthGoal, fragAlign));+ highlighter.setBreakIterator(+ () ->+ LengthGoalBreakIterator.createClosestToLength(+ new CustomSeparatorBreakIterator(separator), lengthGoal, fragAlign));return highlighter.highlightWithoutSearcher(FIELD, query, content, maxPassages).toString();}- private String highlightMinLen(String content, Query query, int lengthGoal, float fragAlign) throws IOException {+ private String highlightMinLen(String content, Query query, int lengthGoal, float fragAlign)+ throws IOException {return highlightMinLen(content, query, lengthGoal, fragAlign, '.');}- private String highlightMinLen(String content, Query query, int lengthGoal, float fragAlign, char separator) throws IOException {+ private String highlightMinLen(+ String content, Query query, int lengthGoal, float fragAlign, char separator)+ throws IOException {// differs from above only by "createMinLength"UnifiedHighlighter highlighter = new UnifiedHighlighter(null, analyzer);- highlighter.setBreakIterator(() -> LengthGoalBreakIterator.createMinLength(new CustomSeparatorBreakIterator(separator), lengthGoal, fragAlign));+ highlighter.setBreakIterator(+ () ->+ LengthGoalBreakIterator.createMinLength(+ new CustomSeparatorBreakIterator(separator), lengthGoal, fragAlign));return highlighter.highlightWithoutSearcher(FIELD, query, content, 1).toString();}-}\ No newline at end of file+}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestSplittingBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestSplittingBreakIterator.javaindex 2fd305c96ef..e09056d6d61 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestSplittingBreakIterator.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestSplittingBreakIterator.java@@ -21,12 +21,10 @@ import java.util.ArrayList;import java.util.Collections;import java.util.List;import java.util.Locale;-import org.apache.lucene.util.LuceneTestCase;public class TestSplittingBreakIterator extends LuceneTestCase {-private static final BreakIterator LINE_BI = BreakIterator.getLineInstance(Locale.ROOT);private static final BreakIterator SPLIT_BI = new SplittingBreakIterator(LINE_BI, '|');@@ -36,15 +34,9 @@ public class TestSplittingBreakIterator extends LuceneTestCase {private void testWithoutSplits(BreakIterator bi) {// these tests have no '|'- testBreakIterator(bi,- " a",- "^^^");- testBreakIterator(bi,- "aa",- "^ ^");- testBreakIterator(bi,- "aa a",- "^ ^^");+ testBreakIterator(bi, " a", "^^^");+ testBreakIterator(bi, "aa", "^ ^");+ testBreakIterator(bi, "aa a", "^ ^^");}public void testWithoutSplits() {@@ -52,57 +44,45 @@ public class TestSplittingBreakIterator extends LuceneTestCase {}public void testOnlySingleSplitChar() {- testBreakIterator(SPLIT_BI,- "|",- "^^");+ testBreakIterator(SPLIT_BI, "|", "^^");}public void testSplitThenValue() {- testBreakIterator(SPLIT_BI,- "|a",- "^^^");+ testBreakIterator(SPLIT_BI, "|a", "^^^");}public void testValueThenSplit() {- testBreakIterator(SPLIT_BI,- "a|",- "^^^");+ testBreakIterator(SPLIT_BI, "a|", "^^^");}public void testValueThenSplitThenValue() {- testBreakIterator(SPLIT_BI,- "aa|aa",- "^ ^^ ^");+ testBreakIterator(SPLIT_BI, "aa|aa", "^ ^^ ^");}public void testValueThenDoubleSplitThenValue() {- testBreakIterator(SPLIT_BI,- "aa||aa",- "^ ^^^ ^");+ testBreakIterator(SPLIT_BI, "aa||aa", "^ ^^^ ^");}public void testValueThenSplitThenDoubleValueThenSplitThenValue() {- testBreakIterator(SPLIT_BI,- "a|bb cc|d",- "^^^ ^ ^^^");+ testBreakIterator(SPLIT_BI, "a|bb cc|d", "^^^ ^ ^^^");}private void testBreakIterator(BreakIterator bi, String text, String boundaries) {bi.setText(text);- //Test first & last+ // Test first & lasttestFirstAndLast(bi, text, boundaries);- //Test if expected boundaries are consistent with reading them from next() in a loop:+ // Test if expected boundaries are consistent with reading them from next() in a loop:assertEquals(boundaries, readBoundariesToString(bi, text));- //Test following() and preceding():+ // Test following() and preceding():// get each index, randomized in case their is a sequencing bug:List<Integer> indexes = randomIntsBetweenInclusive(text.length() + 1);testFollowing(bi, text, boundaries, indexes);testPreceding(bi, text, boundaries, indexes);- //Test previous():+ // Test previous():testPrevious(bi, text, boundaries);}@@ -117,7 +97,8 @@ public class TestSplittingBreakIterator extends LuceneTestCase {assertEquals(message, current, bi.current());}- private void testFollowing(BreakIterator bi, String text, String boundaries, List<Integer> indexes) {+ private void testFollowing(+ BreakIterator bi, String text, String boundaries, List<Integer> indexes) {String message = "Text: " + text;for (Integer index : indexes) {int got = bi.following(index);@@ -130,7 +111,8 @@ public class TestSplittingBreakIterator extends LuceneTestCase {}}- private void testPreceding(BreakIterator bi, String text, String boundaries, List<Integer> indexes) {+ private void testPreceding(+ BreakIterator bi, String text, String boundaries, List<Integer> indexes) {String message = "Text: " + text;for (Integer index : indexes) {int got = bi.preceding(index);@@ -139,9 +121,9 @@ public class TestSplittingBreakIterator extends LuceneTestCase {assertEquals(boundaries.indexOf('^'), bi.current());continue;}-// if (index == text.length() && got == BreakIterator.DONE) {-// continue;//hack to accept faulty default impl of BreakIterator.preceding()-// }+ // if (index == text.length() && got == BreakIterator.DONE) {+ // continue;//hack to accept faulty default impl of BreakIterator.preceding()+ // }assertEquals(message + " index:" + index, boundaries.lastIndexOf('^', index - 1), got);}}@@ -159,7 +141,7 @@ public class TestSplittingBreakIterator extends LuceneTestCase {String message = "Text: " + text;bi.setText(text);- int idx = bi.last();//position at the end+ int idx = bi.last(); // position at the endwhile (true) {idx = boundaries.lastIndexOf('^', idx - 1);if (idx == -1) {@@ -168,12 +150,10 @@ public class TestSplittingBreakIterator extends LuceneTestCase {}assertEquals(message, idx, bi.previous());}- assertEquals(message, boundaries.indexOf('^'), bi.current());//finishes at first+ assertEquals(message, boundaries.indexOf('^'), bi.current()); // finishes at first}- /**- * Returns a string comprised of spaces and '^' only at the boundaries.- */+ /** Returns a string comprised of spaces and '^' only at the boundaries. */private String readBoundariesToString(BreakIterator bi, String text) {// init markers to spacesStringBuilder markers = new StringBuilder();diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.javaindex d91d25b27fd..f91f040c783 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java@@ -16,6 +16,7 @@*/package org.apache.lucene.search.uhighlight;+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;@@ -28,8 +29,6 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.function.Predicate;--import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -63,7 +62,8 @@ import org.junit.Before;public class TestUnifiedHighlighter extends LuceneTestCase {- private final FieldType fieldType; // for "body" generally, but not necessarily others. See constructor+ private final FieldType+ fieldType; // for "body" generally, but not necessarily others. See constructorprivate MockAnalyzer indexAnalyzer;private Directory dir;@@ -79,7 +79,9 @@ public class TestUnifiedHighlighter extends LuceneTestCase {@Beforepublic void doBefore() throws IOException {- indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase+ indexAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercasedir = newDirectory();}@@ -88,44 +90,52 @@ public class TestUnifiedHighlighter extends LuceneTestCase {dir.close();}- static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer) {- return randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.noneOf(HighlightFlag.class), null);+ static UnifiedHighlighter randomUnifiedHighlighter(+ IndexSearcher searcher, Analyzer indexAnalyzer) {+ return randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.noneOf(HighlightFlag.class), null);}- static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer,- EnumSet<HighlightFlag> mandatoryFlags, Boolean requireFieldMatch) {- final UnifiedHighlighter uh = new UnifiedHighlighter(searcher, indexAnalyzer) {- Set<HighlightFlag> flags; // consistently random set of flags for this test run- @Override- protected Set<HighlightFlag> getFlags(String field) {- if (flags != null) {- return flags;- }- final EnumSet<HighlightFlag> result = EnumSet.copyOf(mandatoryFlags);- int r = random().nextInt();- for (HighlightFlag highlightFlag : HighlightFlag.values()) {- if (((1 << highlightFlag.ordinal()) & r) == 0) {- result.add(highlightFlag);+ static UnifiedHighlighter randomUnifiedHighlighter(+ IndexSearcher searcher,+ Analyzer indexAnalyzer,+ EnumSet<HighlightFlag> mandatoryFlags,+ Boolean requireFieldMatch) {+ final UnifiedHighlighter uh =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ Set<HighlightFlag> flags; // consistently random set of flags for this test run++ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ if (flags != null) {+ return flags;+ }+ final EnumSet<HighlightFlag> result = EnumSet.copyOf(mandatoryFlags);+ int r = random().nextInt();+ for (HighlightFlag highlightFlag : HighlightFlag.values()) {+ if (((1 << highlightFlag.ordinal()) & r) == 0) {+ result.add(highlightFlag);+ }+ }+ if (result.contains(HighlightFlag.WEIGHT_MATCHES)) {+ // these two are required for WEIGHT_MATCHES+ result.add(HighlightFlag.MULTI_TERM_QUERY);+ result.add(HighlightFlag.PHRASES);+ }+ return flags = result;}- }- if (result.contains(HighlightFlag.WEIGHT_MATCHES)) {- // these two are required for WEIGHT_MATCHES- result.add(HighlightFlag.MULTI_TERM_QUERY);- result.add(HighlightFlag.PHRASES);- }- return flags = result;- }- };+ };uh.setCacheFieldValCharsThreshold(random().nextInt(100));- if (requireFieldMatch == Boolean.FALSE || (requireFieldMatch == null && random().nextBoolean())) {+ if (requireFieldMatch == Boolean.FALSE+ || (requireFieldMatch == null && random().nextBoolean())) {uh.setFieldMatcher(f -> true); // requireFieldMatch==false}return uh;-}//- // Tests below were ported from the PostingsHighlighter. Possibly augmented. Far below are newer tests.+ // Tests below were ported from the PostingsHighlighter. Possibly augmented. Far below are newer+ // tests.//public void testBasics() throws Exception {@@ -135,7 +145,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -214,7 +225,6 @@ public class TestUnifiedHighlighter extends LuceneTestCase {highlighter.setMaxLength(maxLength);String snippets[] = highlighter.highlight("body", query, topDocs);-ir.close();return snippets;}@@ -313,7 +323,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {doc.add(body);doc.add(title);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");title.setStringValue("I am hoping for the best.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");@@ -325,13 +336,15 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("title", "best")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("title", "best")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);- Map<String, String[]> snippets = highlighter.highlightFields(new String[]{"body", "title"}, query, topDocs);+ Map<String, String[]> snippets =+ highlighter.highlightFields(new String[] {"body", "title"}, query, topDocs);assertEquals(2, snippets.size());assertEquals("Just a test <b>highlighting</b> from postings. ", snippets.get("body")[0]);assertEquals("<b>Highlighting</b> the first term. ", snippets.get("body")[1]);@@ -347,7 +360,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -357,11 +371,12 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "just")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "first")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "just")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "first")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -379,7 +394,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("This test is another test. Not a good sentence. Test test test test.");iw.addDocument(doc);@@ -394,22 +410,26 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(2, snippets.length);- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);- assertEquals("This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[1]);+ assertEquals(+ "This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);+ assertEquals(+ "This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.",+ snippets[1]);ir.close();}public void testBuddhism() throws Exception {- String text = "This eight-volume set brings together seminal papers in Buddhist studies from a vast " +- "range of academic disciplines published over the last forty years. With a new introduction " +- "by the editor, this collection is a unique and unrivalled research resource for both " +- "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in " +- "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine " +- "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma " +- "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric " +- "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and " +- "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";+ String text =+ "This eight-volume set brings together seminal papers in Buddhist studies from a vast "+ + "range of academic disciplines published over the last forty years. With a new introduction "+ + "by the editor, this collection is a unique and unrivalled research resource for both "+ + "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in "+ + "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine "+ + "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma "+ + "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric "+ + "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and "+ + "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Field body = new Field("body", text, fieldType);@@ -419,17 +439,20 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- PhraseQuery query = new PhraseQuery.Builder()- .add(new Term("body", "buddhist"))- .add(new Term("body", "origins"))- .build();+ PhraseQuery query =+ new PhraseQuery.Builder()+ .add(new Term("body", "buddhist"))+ .add(new Term("body", "origins"))+ .build();TopDocs topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);highlighter.setHighlightPhrasesStrictly(false);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(1, snippets.length);- if (highlighter.getFlags("body").containsAll(EnumSet.of(HighlightFlag.WEIGHT_MATCHES, HighlightFlag.PHRASES))) {+ if (highlighter+ .getFlags("body")+ .containsAll(EnumSet.of(HighlightFlag.WEIGHT_MATCHES, HighlightFlag.PHRASES))) {assertTrue(snippets[0], snippets[0].contains("<b>Buddhist origins</b>"));} else {assertTrue(snippets[0], snippets[0].contains("<b>Buddhist</b> <b>origins</b>"));@@ -438,10 +461,11 @@ public class TestUnifiedHighlighter extends LuceneTestCase {}public void testCuriousGeorge() throws Exception {- String text = "It’s the formula for success for preschoolers—Curious George and fire trucks! " +- "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s " +- "popular primate and painted in the original watercolor and charcoal style. " +- "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";+ String text =+ "It’s the formula for success for preschoolers—Curious George and fire trucks! "+ + "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s "+ + "popular primate and painted in the original watercolor and charcoal style. "+ + "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Field body = new Field("body", text, fieldType);@@ -451,10 +475,11 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- PhraseQuery query = new PhraseQuery.Builder()- .add(new Term("body", "curious"))- .add(new Term("body", "george"))- .build();+ PhraseQuery query =+ new PhraseQuery.Builder()+ .add(new Term("body", "curious"))+ .add(new Term("body", "george"))+ .build();TopDocs topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);@@ -466,8 +491,10 @@ public class TestUnifiedHighlighter extends LuceneTestCase {}public void testCambridgeMA() throws Exception {- BufferedReader r = new BufferedReader(new InputStreamReader(- this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));+ BufferedReader r =+ new BufferedReader(+ new InputStreamReader(+ this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));String text = r.readLine();r.close();RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);@@ -478,11 +505,12 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "porter")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "square")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "massachusetts")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "porter")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "square")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "massachusetts")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);@@ -501,7 +529,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");+ body.setStringValue(+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");iw.addDocument(doc);IndexReader ir = iw.getReader();@@ -514,7 +543,9 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(1, snippets.length);- assertEquals("This is a <b>test</b>. ... Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);+ assertEquals(+ "This is a <b>test</b>. ... Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.",+ snippets[0]);ir.close();}@@ -522,7 +553,9 @@ public class TestUnifiedHighlighter extends LuceneTestCase {public void testBooleanMustNot() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);- Field body = new Field("body", "This sentence has both terms. This sentence has only terms.", fieldType);+ Field body =+ new Field(+ "body", "This sentence has both terms. This sentence has only terms.", fieldType);Document document = new Document();document.add(body);iw.addDocument(document);@@ -530,14 +563,16 @@ public class TestUnifiedHighlighter extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- BooleanQuery query2 = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "both")), BooleanClause.Occur.MUST_NOT)- .build();+ BooleanQuery query2 =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "both")), BooleanClause.Occur.MUST_NOT)+ .build();- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "terms")), BooleanClause.Occur.SHOULD)- .add(query2, BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "terms")), BooleanClause.Occur.SHOULD)+ .add(query2, BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);@@ -556,26 +591,30 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");+ body.setStringValue(+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };highlighter.setMaxLength(10000);Query query = new TermQuery(new Term("body", "test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(1, snippets.length);- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);+ assertEquals(+ "This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.",+ snippets[0]);ir.close();}@@ -587,7 +626,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -604,7 +644,10 @@ public class TestUnifiedHighlighter extends LuceneTestCase {int[] docIDs = new int[2];docIDs[0] = hits[0].doc;docIDs[1] = hits[1].doc;- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{1}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {1})+ .get("body");assertEquals(2, snippets.length);assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);@@ -617,7 +660,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();- final String text = "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";+ final String text =+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";Field body = new Field("body", text, fieldType);doc.add(body);iw.addDocument(doc);@@ -627,42 +671,47 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected List<CharSequence[]> loadFieldValues(String[] fields,- DocIdSetIterator docIter, int cacheCharsThreshold) throws IOException {- assert fields.length == 1;- assert docIter.cost() == 1;- docIter.nextDoc();- return Collections.singletonList(new CharSequence[]{text});- }+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected List<CharSequence[]> loadFieldValues(+ String[] fields, DocIdSetIterator docIter, int cacheCharsThreshold)+ throws IOException {+ assert fields.length == 1;+ assert docIter.cost() == 1;+ docIter.nextDoc();+ return Collections.singletonList(new CharSequence[] {text});+ }- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };Query query = new TermQuery(new Term("body", "test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(1, snippets.length);- assertEquals("This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[0]);+ assertEquals(+ "This is a <b>test</b>. Just highlighting from postings. This is also a much sillier <b>test</b>. Feel free to <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b> <b>test</b>.",+ snippets[0]);ir.close();}- /**- * Make sure highlighter returns first N sentences if- * there were no hits.- */+ /** Make sure highlighter returns first N sentences if there were no hits. */public void testEmptyHighlights() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -672,22 +721,23 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);Query query = new TermQuery(new Term("body", "highlighting"));- int[] docIDs = new int[]{0};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ int[] docIDs = new int[] {0};+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals("test this is. another sentence this test has. ", snippets[0]);ir.close();}- /**- * Not empty but nothing analyzes. Ensures we address null term-vectors.- */+ /** Not empty but nothing analyzes. Ensures we address null term-vectors. */public void testNothingAnalyzes() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- doc.add(new Field("body", " ", fieldType));// just a space! (thus not empty)+ doc.add(new Field("body", " ", fieldType)); // just a space! (thus not empty)doc.add(newTextField("id", "id", Field.Store.YES));iw.addDocument(doc);@@ -705,24 +755,28 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Query query = new TermQuery(new Term("body", "highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals(" ", snippets[0]);ir.close();}- /**- * Make sure highlighter we can customize how emtpy- * highlight is returned.- */+ /** Make sure highlighter we can customize how emtpy highlight is returned. */public void testCustomEmptyHighlights() throws Exception {indexAnalyzer.setPositionIncrementGap(10);RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -731,26 +785,30 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setMaxNoHighlightPassages(0);// don't want any default summary+ highlighter.setMaxNoHighlightPassages(0); // don't want any default summaryQuery query = new TermQuery(new Term("body", "highlighting"));- int[] docIDs = new int[]{0};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ int[] docIDs = new int[] {0};+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertNull(snippets[0]);ir.close();}- /**- * Make sure highlighter returns whole text when there- * are no hits and BreakIterator is null.- */+ /** Make sure highlighter returns whole text when there are no hits and BreakIterator is null. */public void testEmptyHighlightsWhole() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -758,31 +816,37 @@ public class TestUnifiedHighlighter extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };Query query = new TermQuery(new Term("body", "highlighting"));- int[] docIDs = new int[]{0};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ int[] docIDs = new int[] {0};+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);- assertEquals("test this is. another sentence this test has. far away is that planet.", snippets[0]);+ assertEquals(+ "test this is. another sentence this test has. far away is that planet.", snippets[0]);ir.close();}- /**- * Make sure highlighter is OK with entirely missing- * field.- */+ /** Make sure highlighter is OK with entirely missing field. */public void testFieldIsMissing() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -792,8 +856,11 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);Query query = new TermQuery(new Term("bogus", "highlighting"));- int[] docIDs = new int[]{0};- String snippets[] = highlighter.highlightFields(new String[]{"bogus"}, query, docIDs, new int[]{2}).get("bogus");+ int[] docIDs = new int[] {0};+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"bogus"}, query, docIDs, new int[] {2})+ .get("bogus");assertEquals(1, snippets.length);assertNull(snippets[0]);@@ -822,7 +889,10 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Query query = new TermQuery(new Term("body", "highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals(" ", snippets[0]);@@ -851,7 +921,10 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Query query = new TermQuery(new Term("body", "highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertNull(snippets[0]);@@ -882,7 +955,8 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setCacheFieldValCharsThreshold(random().nextInt(10) * 10);// 0 thru 90 intervals of 10+ highlighter.setCacheFieldValCharsThreshold(+ random().nextInt(10) * 10); // 0 thru 90 intervals of 10Query query = new TermQuery(new Term("body", "answer"));TopDocs hits = searcher.search(query, numDocs);assertEquals(numDocs, hits.totalHits.value);@@ -911,8 +985,10 @@ public class TestUnifiedHighlighter extends LuceneTestCase {doc.add(body);doc.add(title);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");- title.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");+ title.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();@@ -920,15 +996,19 @@ public class TestUnifiedHighlighter extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD)- .build();- Map<String, String[]> snippets = highlighter.highlightFields(new String[]{"title", "body"}, query, new int[]{0}, new int[]{1, 2});+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD)+ .build();+ Map<String, String[]> snippets =+ highlighter.highlightFields(+ new String[] {"title", "body"}, query, new int[] {0}, new int[] {1, 2});String titleHighlight = snippets.get("title")[0];String bodyHighlight = snippets.get("body")[0];assertEquals("This is a <b>test</b>. ", titleHighlight);- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", bodyHighlight);+ assertEquals(+ "This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", bodyHighlight);ir.close();}@@ -939,25 +1019,28 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- return new DefaultPassageFormatter("<b>", "</b>", "... ", true);- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected PassageFormatter getFormatter(String field) {+ return new DefaultPassageFormatter("<b>", "</b>", "... ", true);+ }+ };Query query = new TermQuery(new Term("body", "highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);assertEquals(1, snippets.length);- assertEquals("Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);+ assertEquals(+ "Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);ir.close();}@@ -970,38 +1053,46 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- return new PassageFormatter() {- PassageFormatter defaultFormatter = new DefaultPassageFormatter();-+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {@Override- public String[] format(Passage passages[], String content) {- // Just turns the String snippet into a length 2- // array of String- return new String[]{"blah blah", defaultFormatter.format(passages, content).toString()};+ protected PassageFormatter getFormatter(String field) {+ return new PassageFormatter() {+ PassageFormatter defaultFormatter = new DefaultPassageFormatter();++ @Override+ public String[] format(Passage passages[], String content) {+ // Just turns the String snippet into a length 2+ // array of String+ return new String[] {+ "blah blah", defaultFormatter.format(passages, content).toString()+ };+ }+ };}};- }- };Query query = new TermQuery(new Term("body", "highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);int[] docIDs = new int[1];docIDs[0] = topDocs.scoreDocs[0].doc;- Map<String, Object[]> snippets = highlighter.highlightFieldsAsObjects(new String[]{"body"}, query, docIDs, new int[]{1});+ Map<String, Object[]> snippets =+ highlighter.highlightFieldsAsObjects(new String[] {"body"}, query, docIDs, new int[] {1});Object[] bodySnippets = snippets.get("body");assertEquals(1, bodySnippets.length);- assertTrue(Arrays.equals(new String[]{"blah blah", "Just a test <b>highlighting</b> from postings. "}, (String[]) bodySnippets[0]));+ assertTrue(+ Arrays.equals(+ new String[] {"blah blah", "Just a test <b>highlighting</b> from postings. "},+ (String[]) bodySnippets[0]));ir.close();}@@ -1035,15 +1126,16 @@ public class TestUnifiedHighlighter extends LuceneTestCase {public void testFieldMatcherTermQuery() throws Exception {IndexReader ir = indexSomeFields();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighterNoFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Predicate<String> getFieldMatcher(String field) {- // requireFieldMatch=false- return (qf) -> true;- }- };+ UnifiedHighlighter highlighterNoFieldMatch =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Predicate<String> getFieldMatcher(String field) {+ // requireFieldMatch=false+ return (qf) -> true;+ }+ };UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighterFieldMatch.setFieldMatcher(null);//default+ highlighterFieldMatch.setFieldMatcher(null); // defaultBooleanQuery.Builder queryBuilder =new BooleanQuery.Builder().add(new TermQuery(new Term("text", "some")), BooleanClause.Occur.SHOULD)@@ -1081,11 +1173,15 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighterNoFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);- assertEquals("<b>This</b> <b>is</b> the text <b>field</b>. You can put <b>some</b> text if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> <b>is</b> the text <b>field</b>. You can put <b>some</b> text if you want.",+ snippets[0]);snippets = highlighterFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);- assertEquals("<b>This</b> is the text <b>field</b>. You can put <b>some</b> text if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> is the text <b>field</b>. You can put <b>some</b> text if you want.",+ snippets[0]);highlighterFieldMatch.setFieldMatcher((fq) -> "title".equals(fq));snippets = highlighterFieldMatch.highlight("text", query, topDocs, 10);@@ -1106,7 +1202,6 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(1, snippets.length);assertEquals("<b>This</b> is the <b>category</b> field.", snippets[0]);-highlighterFieldMatch.setFieldMatcher((fq) -> "title".equals(fq));snippets = highlighterFieldMatch.highlight("category", query, topDocs, 10);assertEquals(1, snippets.length);@@ -1119,15 +1214,18 @@ public class TestUnifiedHighlighter extends LuceneTestCase {public void testFieldMatcherMultiTermQuery() throws Exception {IndexReader ir = indexSomeFields();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighterNoFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Predicate<String> getFieldMatcher(String field) {- // requireFieldMatch=false- return (qf) -> true;- }- };- UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.of(HighlightFlag.MULTI_TERM_QUERY), null);- highlighterFieldMatch.setFieldMatcher(null);//default+ UnifiedHighlighter highlighterNoFieldMatch =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Predicate<String> getFieldMatcher(String field) {+ // requireFieldMatch=false+ return (qf) -> true;+ }+ };+ UnifiedHighlighter highlighterFieldMatch =+ randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.of(HighlightFlag.MULTI_TERM_QUERY), null);+ highlighterFieldMatch.setFieldMatcher(null); // defaultBooleanQuery.Builder queryBuilder =new BooleanQuery.Builder().add(new FuzzyQuery(new Term("text", "sime"), 1), BooleanClause.Occur.SHOULD)@@ -1165,11 +1263,15 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighterNoFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);- assertEquals("<b>This</b> <b>is</b> the text <b>field</b>. You can put <b>some</b> text if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> <b>is</b> the text <b>field</b>. You can put <b>some</b> text if you want.",+ snippets[0]);snippets = highlighterFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);- assertEquals("<b>This</b> is the text <b>field</b>. You can put <b>some</b> text if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> is the text <b>field</b>. You can put <b>some</b> text if you want.",+ snippets[0]);highlighterFieldMatch.setFieldMatcher((fq) -> "title".equals(fq));snippets = highlighterFieldMatch.highlight("text", query, topDocs, 10);@@ -1190,7 +1292,6 @@ public class TestUnifiedHighlighter extends LuceneTestCase {assertEquals(1, snippets.length);assertEquals("<b>This</b> is the <b>category</b> field.", snippets[0]);-highlighterFieldMatch.setFieldMatcher((fq) -> "title".equals(fq));snippets = highlighterFieldMatch.highlight("category", query, topDocs, 10);assertEquals(1, snippets.length);@@ -1220,22 +1321,32 @@ public class TestUnifiedHighlighter extends LuceneTestCase {public void testFieldMatcherPhraseQuery() throws Exception {IndexReader ir = indexSomeFields();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighterNoFieldMatch = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Predicate<String> getFieldMatcher(String field) {- // requireFieldMatch=false- return (qf) -> true;- }- };- UnifiedHighlighter highlighterFieldMatch = randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.of(HighlightFlag.PHRASES, HighlightFlag.MULTI_TERM_QUERY), null);- highlighterFieldMatch.setFieldMatcher(null);//default+ UnifiedHighlighter highlighterNoFieldMatch =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Predicate<String> getFieldMatcher(String field) {+ // requireFieldMatch=false+ return (qf) -> true;+ }+ };+ UnifiedHighlighter highlighterFieldMatch =+ randomUnifiedHighlighter(+ searcher,+ indexAnalyzer,+ EnumSet.of(HighlightFlag.PHRASES, HighlightFlag.MULTI_TERM_QUERY),+ null);+ highlighterFieldMatch.setFieldMatcher(null); // defaultBooleanQuery.Builder queryBuilder =new BooleanQuery.Builder().add(new PhraseQuery("title", "this", "is", "the", "title"), BooleanClause.Occur.SHOULD)- .add(new PhraseQuery(2, "category", "this", "is", "the", "field"), BooleanClause.Occur.SHOULD)+ .add(+ new PhraseQuery(2, "category", "this", "is", "the", "field"),+ BooleanClause.Occur.SHOULD).add(new PhraseQuery("text", "this", "is"), BooleanClause.Occur.SHOULD).add(new PhraseQuery("category", "this", "is"), BooleanClause.Occur.SHOULD)- .add(new PhraseQuery(1, "text", "you", "can", "put", "text"), BooleanClause.Occur.SHOULD);+ .add(+ new PhraseQuery(1, "text", "you", "can", "put", "text"),+ BooleanClause.Occur.SHOULD);Query query = queryBuilder.build();// title@@ -1276,19 +1387,28 @@ public class TestUnifiedHighlighter extends LuceneTestCase {String[] snippets = highlighterNoFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);if (highlighterNoFieldMatch.getFlags("text").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertEquals("<b>This is the text field</b>. <b>You can put some text</b> if you want.", snippets[0]);+ assertEquals(+ "<b>This is the text field</b>. <b>You can put some text</b> if you want.",+ snippets[0]);} else {- assertEquals("<b>This</b> <b>is</b> <b>the</b> <b>text</b> <b>field</b>. <b>You</b> <b>can</b> <b>put</b> some <b>text</b> if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> <b>is</b> <b>the</b> <b>text</b> <b>field</b>. <b>You</b> <b>can</b> <b>put</b> some <b>text</b> if you want.",+ snippets[0]);}snippets = highlighterFieldMatch.highlight("text", query, topDocs, 10);assertEquals(1, snippets.length);if (highlighterFieldMatch.getFlags("text").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertEquals("<b>This is</b> the text field. <b>You can put some text</b> if you want.", snippets[0]);+ assertEquals(+ "<b>This is</b> the text field. <b>You can put some text</b> if you want.",+ snippets[0]);} else {- // note: odd that the first "text" is highlighted. Apparently WSTE converts PhraseQuery to a SpanNearQuery with+ // note: odd that the first "text" is highlighted. Apparently WSTE converts PhraseQuery to+ // a SpanNearQuery with// with inorder=false when non-0 slop. Probably a bug.- assertEquals("<b>This</b> <b>is</b> the <b>text</b> field. <b>You</b> <b>can</b> <b>put</b> some <b>text</b> if you want.", snippets[0]);+ assertEquals(+ "<b>This</b> <b>is</b> the <b>text</b> field. <b>You</b> <b>can</b> <b>put</b> some <b>text</b> if you want.",+ snippets[0]);}highlighterFieldMatch.setFieldMatcher((fq) -> "title".equals(fq));@@ -1335,28 +1455,35 @@ public class TestUnifiedHighlighter extends LuceneTestCase {public void testNestedBooleanQueryAccuracy() throws IOException {IndexReader ir = indexSomeFields();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer,- EnumSet.of(HighlightFlag.WEIGHT_MATCHES), true);+ UnifiedHighlighter highlighter =+ randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.of(HighlightFlag.WEIGHT_MATCHES), true);// This query contains an inner Boolean of two MUST clauses (== "AND"). Only one of them is- // actually in the data, the other is not. We should highlight neither. We can highlight the outer+ // actually in the data, the other is not. We should highlight neither. We can highlight the+ // outer// SHOULD clauses though.- Query query = new BooleanQuery.Builder()- .add(new PhraseQuery("title", "title", "field"), BooleanClause.Occur.SHOULD)- .add(new BooleanQuery.Builder()- .add(new TermQuery(new Term("category", "category")), BooleanClause.Occur.MUST)- .add(new TermQuery(new Term("category", "nonexistent")), BooleanClause.Occur.MUST)- .build(), BooleanClause.Occur.SHOULD)- .build();+ Query query =+ new BooleanQuery.Builder()+ .add(new PhraseQuery("title", "title", "field"), BooleanClause.Occur.SHOULD)+ .add(+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("category", "category")), BooleanClause.Occur.MUST)+ .add(+ new TermQuery(new Term("category", "nonexistent")),+ BooleanClause.Occur.MUST)+ .build(),+ BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("title", query, topDocs);- assertArrayEquals(new String[]{"This is the <b>title field</b>."}, snippets);+ assertArrayEquals(new String[] {"This is the <b>title field</b>."}, snippets);// no highlights, not of "category" since "nonexistent" wasn't theresnippets = highlighter.highlight("category", query, topDocs);- assertArrayEquals(new String[]{"This is the category field."}, snippets);+ assertArrayEquals(new String[] {"This is the category field."}, snippets);ir.close();}@@ -1372,19 +1499,23 @@ public class TestUnifiedHighlighter extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- throw new AssertionError("shouldn't be called");- }- });+ UnifiedHighlighter highlighter =+ randomUnifiedHighlighter(+ searcher,+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ throw new AssertionError("shouldn't be called");+ }+ });Query query = new TermQuery(new Term("body", "highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);@@ -1409,49 +1540,56 @@ public class TestUnifiedHighlighter extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer,- EnumSet.of(HighlightFlag.WEIGHT_MATCHES), null);- Query query = new BooleanQuery.Builder()- // simple term query body:one- .add(new TermQuery(new Term(body.name(), "one")), BooleanClause.Occur.MUST)- // a custom query, a leaf, that which matches body:sentence- // Note this isn't even an MTQ. What matters is that Weight.matches works.- .add(new Query() {- @Override- public String toString(String field) {- return "bogus";- }-- @Override- public Query rewrite(IndexReader reader) {- return this;- }-- // we don't visit terms, and we don't expose an automata. Thus this appears as some unknown leaf.- @Override- public void visit(QueryVisitor visitor) {- if (visitor.acceptField(body.name())) {- visitor.visitLeaf(this);- }- }-- @Override- public boolean equals(Object obj) {- return this == obj;- }-- @Override- public int hashCode() {- return System.identityHashCode(this);- }-- @Override- public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {- //TODO maybe should loop through index terms to show we can see other terms- return new TermQuery(new Term(body.name(), "sentence")).createWeight(searcher, scoreMode, boost);- }- }, BooleanClause.Occur.MUST)- .build();+ UnifiedHighlighter highlighter =+ randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.of(HighlightFlag.WEIGHT_MATCHES), null);+ Query query =+ new BooleanQuery.Builder()+ // simple term query body:one+ .add(new TermQuery(new Term(body.name(), "one")), BooleanClause.Occur.MUST)+ // a custom query, a leaf, that which matches body:sentence+ // Note this isn't even an MTQ. What matters is that Weight.matches works.+ .add(+ new Query() {+ @Override+ public String toString(String field) {+ return "bogus";+ }++ @Override+ public Query rewrite(IndexReader reader) {+ return this;+ }++ // we don't visit terms, and we don't expose an automata. Thus this appears as+ // some unknown leaf.+ @Override+ public void visit(QueryVisitor visitor) {+ if (visitor.acceptField(body.name())) {+ visitor.visitLeaf(this);+ }+ }++ @Override+ public boolean equals(Object obj) {+ return this == obj;+ }++ @Override+ public int hashCode() {+ return System.identityHashCode(this);+ }++ @Override+ public Weight createWeight(+ IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {+ // TODO maybe should loop through index terms to show we can see other terms+ return new TermQuery(new Term(body.name(), "sentence"))+ .createWeight(searcher, scoreMode, boost);+ }+ },+ BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighter.highlight("body", query, topDocs);diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.javaindex e0b5debf95d..f70ce3e87f2 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterMTQ.java@@ -16,15 +16,13 @@*/package org.apache.lucene.search.uhighlight;-+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import com.carrotsearch.randomizedtesting.generators.RandomStrings;import java.io.IOException;import java.util.Arrays;import java.util.Collections;import java.util.EnumSet;import java.util.Objects;--import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;-import com.carrotsearch.randomizedtesting.generators.RandomStrings;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -74,9 +72,7 @@ import org.apache.lucene.util.UnicodeUtil;import org.junit.After;import org.junit.Before;-/**- * Some tests that highlight wildcard, fuzzy, etc queries.- */+/** Some tests that highlight wildcard, fuzzy, etc queries. */public class TestUnifiedHighlighterMTQ extends LuceneTestCase {final FieldType fieldType;@@ -96,7 +92,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {@Beforepublic void doBefore() throws IOException {dir = newDirectory();- indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase+ indexAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercase}@After@@ -135,13 +133,14 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals(2, snippets.length);assertEquals("This is a test.", snippets[0]);assertEquals("Test a one sentence document.", snippets[1]);- highlighter.setHandleMultiTermQuery(true);//reset+ highlighter.setHandleMultiTermQuery(true); // reset// wrong field- BooleanQuery bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -152,9 +151,10 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {ir.close();}- private UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer) {- return TestUnifiedHighlighter.randomUnifiedHighlighter(searcher, indexAnalyzer,- EnumSet.of(HighlightFlag.MULTI_TERM_QUERY), null);+ private UnifiedHighlighter randomUnifiedHighlighter(+ IndexSearcher searcher, Analyzer indexAnalyzer) {+ return TestUnifiedHighlighter.randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.of(HighlightFlag.MULTI_TERM_QUERY), null);}public void testOnePrefix() throws Exception {@@ -184,11 +184,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("<b>Test</b> a one sentence document.", snippets[1]);// wrong field- highlighter.setFieldMatcher(null);//default- BooleanQuery bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(new PrefixQuery(new Term("bogus", "te")), BooleanClause.Occur.SHOULD)- .build();+ highlighter.setFieldMatcher(null); // default+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(new PrefixQuery(new Term("bogus", "te")), BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -225,11 +226,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("<b>Test</b> a one sentence document.", snippets[1]);// wrong field- highlighter.setFieldMatcher(null);//default- BooleanQuery bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(new RegexpQuery(new Term("bogus", "te.*")), BooleanClause.Occur.SHOULD)- .build();+ highlighter.setFieldMatcher(null); // default+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(new RegexpQuery(new Term("bogus", "te.*")), BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -284,11 +286,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("<b>Test</b> a one sentence document.", snippets[1]);// wrong field- highlighter.setFieldMatcher(null);//default- BooleanQuery bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(new FuzzyQuery(new Term("bogus", "tets"), 1), BooleanClause.Occur.SHOULD)- .build();+ highlighter.setFieldMatcher(null); // default+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(new FuzzyQuery(new Term("bogus", "tets"), 1), BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -361,10 +364,13 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("<b>Test</b> a one sentence document.", snippets[1]);// exact start exclusive- BooleanQuery bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(TermRangeQuery.newStringRange("body", "test", "tf", false, true), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(+ TermRangeQuery.newStringRange("body", "test", "tf", false, true),+ BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -373,10 +379,13 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("Test a one sentence document.", snippets[1]);// exact end exclusive- bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(TermRangeQuery.newStringRange("body", "ta", "test", true, false), BooleanClause.Occur.SHOULD)- .build();+ bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(+ TermRangeQuery.newStringRange("body", "ta", "test", true, false),+ BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -385,11 +394,14 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("Test a one sentence document.", snippets[1]);// wrong field- highlighter.setFieldMatcher(null);//default- bq = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(TermRangeQuery.newStringRange("bogus", "ta", "tf", true, true), BooleanClause.Occur.SHOULD)- .build();+ highlighter.setFieldMatcher(null); // default+ bq =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(+ TermRangeQuery.newStringRange("bogus", "ta", "tf", true, true),+ BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(bq, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", bq, topDocs);@@ -417,9 +429,10 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- BooleanQuery query = new BooleanQuery.Builder()- .add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -428,10 +441,11 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {assertEquals("<b>Test</b> a one sentence document.", snippets[1]);// must not- query = new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)- .add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.MUST_NOT)- .build();+ query =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD)+ .add(new WildcardQuery(new Term("bogus", "te*")), BooleanClause.Occur.MUST_NOT)+ .build();topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);snippets = highlighter.highlight("body", query, topDocs);@@ -459,10 +473,11 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- BooleanQuery query = new BooleanQuery.Builder()- .add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.MUST)- .add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.FILTER)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new WildcardQuery(new Term("body", "te*")), BooleanClause.Occur.MUST)+ .add(new TermQuery(new Term("body", "test")), BooleanClause.Occur.FILTER)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -518,8 +533,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- DisjunctionMaxQuery query = new DisjunctionMaxQuery(- Collections.singleton(new WildcardQuery(new Term("body", "te*"))), 0);+ DisjunctionMaxQuery query =+ new DisjunctionMaxQuery(+ Collections.singleton(new WildcardQuery(new Term("body", "te*"))), 0);TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -548,8 +564,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);// wrap in a SpanBoostQuery to also show we see inside it- Query query = new SpanBoostQuery(- new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*"))), 2.0f);+ Query query =+ new SpanBoostQuery(+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*"))), 2.0f);TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -577,8 +594,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));- Query query = new SpanOrQuery(new SpanQuery[]{childQuery});+ SpanQuery childQuery =+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));+ Query query = new SpanOrQuery(new SpanQuery[] {childQuery});TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -606,8 +624,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));- Query query = new SpanNearQuery(new SpanQuery[]{childQuery, childQuery}, 0, false);+ SpanQuery childQuery =+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));+ Query query = new SpanNearQuery(new SpanQuery[] {childQuery, childQuery}, 0, false);TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -665,7 +684,8 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));+ SpanQuery childQuery =+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));Query query = new SpanFirstQuery(childQuery, 1000000);TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);@@ -678,8 +698,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {}/**- * Runs a query with two MTQs and confirms the formatter- * can tell which query matched which hit.+ * Runs a query with two MTQs and confirms the formatter can tell which query matched which hit.*/public void testWhichMTQMatched() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);@@ -697,11 +716,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);// use a variety of common MTQ types- BooleanQuery query = new BooleanQuery.Builder()- .add(new PrefixQuery(new Term("body", "te")), BooleanClause.Occur.SHOULD)- .add(new WildcardQuery(new Term("body", "*one*")), BooleanClause.Occur.SHOULD)- .add(new FuzzyQuery(new Term("body", "zentence~")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new PrefixQuery(new Term("body", "te")), BooleanClause.Occur.SHOULD)+ .add(new WildcardQuery(new Term("body", "*one*")), BooleanClause.Occur.SHOULD)+ .add(new FuzzyQuery(new Term("body", "zentence~")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -712,61 +732,63 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {// Now use our own formatter, that also stuffs the// matching term's text into the result:- highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {-- @Override- protected PassageFormatter getFormatter(String field) {- return new PassageFormatter() {+ highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {@Override- public Object format(Passage passages[], String content) {- // Copied from DefaultPassageFormatter, but- // tweaked to include the matched term:- StringBuilder sb = new StringBuilder();- int pos = 0;- for (Passage passage : passages) {- // don't add ellipsis if its the first one, or if its connected.- if (passage.getStartOffset() > pos && pos > 0) {- sb.append("... ");- }- pos = passage.getStartOffset();- for (int i = 0; i < passage.getNumMatches(); i++) {- int start = passage.getMatchStarts()[i];- int end = passage.getMatchEnds()[i];- // its possible to have overlapping terms- if (start > pos) {- sb.append(content, pos, start);- }- if (end > pos) {- sb.append("<b>");- sb.append(content, Math.max(pos, start), end);- sb.append('(');- sb.append(passage.getMatchTerms()[i].utf8ToString());- sb.append(')');- sb.append("</b>");- pos = end;+ protected PassageFormatter getFormatter(String field) {+ return new PassageFormatter() {++ @Override+ public Object format(Passage passages[], String content) {+ // Copied from DefaultPassageFormatter, but+ // tweaked to include the matched term:+ StringBuilder sb = new StringBuilder();+ int pos = 0;+ for (Passage passage : passages) {+ // don't add ellipsis if its the first one, or if its connected.+ if (passage.getStartOffset() > pos && pos > 0) {+ sb.append("... ");+ }+ pos = passage.getStartOffset();+ for (int i = 0; i < passage.getNumMatches(); i++) {+ int start = passage.getMatchStarts()[i];+ int end = passage.getMatchEnds()[i];+ // its possible to have overlapping terms+ if (start > pos) {+ sb.append(content, pos, start);+ }+ if (end > pos) {+ sb.append("<b>");+ sb.append(content, Math.max(pos, start), end);+ sb.append('(');+ sb.append(passage.getMatchTerms()[i].utf8ToString());+ sb.append(')');+ sb.append("</b>");+ pos = end;+ }+ }+ // its possible a "term" from the analyzer could span a sentence boundary.+ sb.append(content, pos, Math.max(pos, passage.getEndOffset()));+ pos = passage.getEndOffset();}+ return sb.toString();}- // its possible a "term" from the analyzer could span a sentence boundary.- sb.append(content, pos, Math.max(pos, passage.getEndOffset()));- pos = passage.getEndOffset();- }- return sb.toString();+ };}};- }- };assertEquals(1, topDocs.totalHits.value);snippets = highlighter.highlight("body", query, topDocs);assertEquals(1, snippets.length);- assertEquals("<b>Test(body:te*)</b> a <b>one(body:*one*)</b> <b>sentence(body:zentence~~2)</b> document.", snippets[0]);+ assertEquals(+ "<b>Test(body:te*)</b> a <b>one(body:*one*)</b> <b>sentence(body:zentence~~2)</b> document.",+ snippets[0]);ir.close();}-//// All tests below were *not* ported from the PostingsHighlighter; they are new to the U.H.//@@ -778,7 +800,8 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("Alpha Bravo foo foo foo. Foo foo Alpha Bravo");//44 char long, 2 sentences+ body.setStringValue(+ "Alpha Bravo foo foo foo. Foo foo Alpha Bravo"); // 44 char long, 2 sentencesiw.addDocument(doc);IndexReader ir = iw.getReader();@@ -786,17 +809,17 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setMaxLength(25);//a little past first sentence+ highlighter.setMaxLength(25); // a little past first sentence- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "alpha")), BooleanClause.Occur.MUST)- .add(new PrefixQuery(new Term("body", "bra")), BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "alpha")), BooleanClause.Occur.MUST)+ .add(new PrefixQuery(new Term("body", "bra")), BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);- String snippets[] = highlighter.highlight("body", query, topDocs, 2);//ask for 2 but we'll only get 1- assertArrayEquals(- new String[]{"<b>Alpha</b> <b>Bravo</b> foo foo foo. "}, snippets- );+ String snippets[] =+ highlighter.highlight("body", query, topDocs, 2); // ask for 2 but we'll only get 1+ assertArrayEquals(new String[] {"<b>Alpha</b> <b>Bravo</b> foo foo foo. "}, snippets);ir.close();}@@ -808,9 +831,10 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {Document doc = new Document();doc.add(body);- //tests interleaving of multiple wildcard matches with the CompositePostingsEnum- //In this case the CompositePostingsEnum will have an underlying PostingsEnum that jumps form pos 1 to 9 for bravo- //and a second with position 2 for Bravado+ // tests interleaving of multiple wildcard matches with the CompositePostingsEnum+ // In this case the CompositePostingsEnum will have an underlying PostingsEnum that jumps form+ // pos 1 to 9 for bravo+ // and a second with position 2 for Bravadobody.setStringValue("Alpha Bravo Bravado foo foo foo. Foo foo Alpha Bravo");iw.addDocument(doc);@@ -819,17 +843,18 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setMaxLength(32);//a little past first sentence+ highlighter.setMaxLength(32); // a little past first sentence- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "alpha")), BooleanClause.Occur.MUST)- .add(new PrefixQuery(new Term("body", "bra")), BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "alpha")), BooleanClause.Occur.MUST)+ .add(new PrefixQuery(new Term("body", "bra")), BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);- String snippets[] = highlighter.highlight("body", query, topDocs, 2);//ask for 2 but we'll only get 1+ String snippets[] =+ highlighter.highlight("body", query, topDocs, 2); // ask for 2 but we'll only get 1assertArrayEquals(- new String[]{"<b>Alpha</b> <b>Bravo</b> <b>Bravado</b> foo foo foo."}, snippets- );+ new String[] {"<b>Alpha</b> <b>Bravo</b> <b>Bravado</b> foo foo foo."}, snippets);ir.close();}@@ -852,24 +877,26 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {iw.close();// use this buggy Analyzer at highlight time- Analyzer buggyAnalyzer = new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer buggyTokenizer = new Tokenizer() {+ Analyzer buggyAnalyzer =+ new Analyzer() {@Override- public boolean incrementToken() throws IOException {- throw new IOException("EXPECTED");+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer buggyTokenizer =+ new Tokenizer() {+ @Override+ public boolean incrementToken() throws IOException {+ throw new IOException("EXPECTED");+ }+ };+ return new TokenStreamComponents(buggyTokenizer);}};- return new TokenStreamComponents(buggyTokenizer);- }- };IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, buggyAnalyzer);highlighter.setHandleMultiTermQuery(true);if (rarely()) {- highlighter.setMaxLength(25);//a little past first sentence+ highlighter.setMaxLength(25); // a little past first sentence}boolean hasClauses = false;@@ -896,22 +923,21 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {}ir.close();- // Now test we can get the tokenStream without it puking due to IllegalStateException for not calling close()+ // Now test we can get the tokenStream without it puking due to IllegalStateException for not+ // calling close()try (TokenStream ts = buggyAnalyzer.tokenStream("body", "anything")) {- ts.reset();// hopefully doesn't throw+ ts.reset(); // hopefully doesn't throw// don't call incrementToken; we know it's buggy ;-)}}- /**- * Not empty but nothing analyzes. Ensures we address null term-vectors.- */+ /** Not empty but nothing analyzes. Ensures we address null term-vectors. */public void testNothingAnalyzes() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- doc.add(new Field("body", " ", fieldType));// just a space! (thus not empty)+ doc.add(new Field("body", " ", fieldType)); // just a space! (thus not empty)doc.add(newTextField("id", "id", Field.Store.YES));iw.addDocument(doc);@@ -929,7 +955,10 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {Query query = new PrefixQuery(new Term("body", "nonexistent"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals(" ", snippets[0]);@@ -958,7 +987,8 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {Query query = new PrefixQuery(new Term("body", "ab"));TopDocs topDocs = searcher.search(query, 10);- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, topDocs).get("body");+ String snippets[] =+ highlighter.highlightFields(new String[] {"body"}, query, topDocs).get("body");Arrays.sort(snippets);assertEquals("[word <b>aberration</b>, word <b>absolve</b>]", Arrays.toString(snippets));@@ -979,19 +1009,24 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);int docID = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;- PhraseQuery pq = new PhraseQuery.Builder()- .add(new Term("body", "consent"))- .add(new Term("body", "order"))- .build();+ PhraseQuery pq =+ new PhraseQuery.Builder()+ .add(new Term("body", "consent"))+ .add(new Term("body", "order"))+ .build();- BooleanQuery query = new BooleanQuery.Builder()- .add(new WildcardQuery(new Term("body", "enforc*")), BooleanClause.Occur.MUST)- .add(pq, BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new WildcardQuery(new Term("body", "enforc*")), BooleanClause.Occur.MUST)+ .add(pq, BooleanClause.Occur.MUST)+ .build();- int[] docIds = new int[]{docID};+ int[] docIds = new int[] {docID};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIds, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIds, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals("iterate insect ipswitch illinois indirect", snippets[0]);ir.close();@@ -1000,7 +1035,9 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {public void testCustomSpanQueryHighlighting() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- doc.add(new Field("body", "alpha bravo charlie delta echo foxtrot golf hotel india juliet", fieldType));+ doc.add(+ new Field(+ "body", "alpha bravo charlie delta echo foxtrot golf hotel india juliet", fieldType));doc.add(newTextField("id", "id", Field.Store.YES));iw.addDocument(doc);@@ -1013,19 +1050,23 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {int docId = searcher.search(new TermQuery(new Term("id", "id")), 1).scoreDocs[0].doc;WildcardQuery wildcardQuery = new WildcardQuery(new Term("body", "foxtr*"));- SpanMultiTermQueryWrapper<WildcardQuery> wildcardQueryWrapper = new SpanMultiTermQueryWrapper<>(wildcardQuery);+ SpanMultiTermQueryWrapper<WildcardQuery> wildcardQueryWrapper =+ new SpanMultiTermQueryWrapper<>(wildcardQuery);SpanQuery wrappedQuery = new MyWrapperSpanQuery(wildcardQueryWrapper);- BooleanQuery query = new BooleanQuery.Builder()- .add(wrappedQuery, BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder().add(wrappedQuery, BooleanClause.Occur.SHOULD).build();- int[] docIds = new int[]{docId};+ int[] docIds = new int[] {docId};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIds, new int[]{2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIds, new int[] {2})+ .get("body");assertEquals(1, snippets.length);- assertEquals("alpha bravo charlie delta echo <b>foxtrot</b> golf hotel india juliet", snippets[0]);+ assertEquals(+ "alpha bravo charlie delta echo <b>foxtrot</b> golf hotel india juliet", snippets[0]);ir.close();}@@ -1044,11 +1085,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {@Overridepublic String toString(String field) {- return "(Wrapper[" + originalQuery.toString(field)+"])";+ return "(Wrapper[" + originalQuery.toString(field) + "])";}@Override- public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {+ public SpanWeight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)+ throws IOException {return originalQuery.createWeight(searcher, scoreMode, boost);}@@ -1061,7 +1103,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {public Query rewrite(IndexReader reader) throws IOException {Query newOriginalQuery = originalQuery.rewrite(reader);if (newOriginalQuery != originalQuery) {- return new MyWrapperSpanQuery((SpanQuery)newOriginalQuery);+ return new MyWrapperSpanQuery((SpanQuery) newOriginalQuery);}return this;}@@ -1070,7 +1112,7 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {public boolean equals(Object o) {if (this == o) return true;if (o == null || getClass() != o.getClass()) return false;- return originalQuery.equals(((MyWrapperSpanQuery)o).originalQuery);+ return originalQuery.equals(((MyWrapperSpanQuery) o).originalQuery);}@Override@@ -1107,13 +1149,12 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {iw.deleteAll();String field = "title";String value = RandomStrings.randomUnicodeOfLength(random(), 3);- if (value.contains(UnifiedHighlighter.MULTIVAL_SEP_CHAR+"")) { // will throw things off+ if (value.contains(UnifiedHighlighter.MULTIVAL_SEP_CHAR + "")) { // will throw things offcontinue;}int[] valuePoints = value.codePoints().toArray();- iw.addDocument(Collections.singleton(- new Field(field, value, fieldType)));+ iw.addDocument(Collections.singleton(new Field(field, value, fieldType)));iw.commit();try (IndexReader ir = iw.getReader()) {IndexSearcher searcher = newSearcher(ir);@@ -1121,15 +1162,11 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {highlighter.setBreakIterator(WholeBreakIterator::new);// Test PrefixQuery- Query query = new PrefixQuery(new Term(field,- UnicodeUtil.newString(valuePoints, 0, 1)));+ Query query = new PrefixQuery(new Term(field, UnicodeUtil.newString(valuePoints, 0, 1)));highlightAndAssertMatch(searcher, highlighter, query, field, value);// Test TermRangeQuery- query = new TermRangeQuery(field,- new BytesRef(value),- new BytesRef(value),- true, true );+ query = new TermRangeQuery(field, new BytesRef(value), new BytesRef(value), true, true);highlightAndAssertMatch(searcher, highlighter, query, field, value);// Test FuzzyQuery@@ -1137,27 +1174,41 @@ public class TestUnifiedHighlighterMTQ extends LuceneTestCase {highlightAndAssertMatch(searcher, highlighter, query, field, value);if (valuePoints.length != 3) {- continue; // even though we ask RandomStrings for a String with 3 code points, it seems sometimes it's less+ continue; // even though we ask RandomStrings for a String with 3 code points, it seems+ // sometimes it's less}// Test WildcardQuery- query = new WildcardQuery(new Term(field,- new StringBuilder()- .append(WildcardQuery.WILDCARD_ESCAPE).appendCodePoint(valuePoints[0])- .append(WildcardQuery.WILDCARD_CHAR)- .append(WildcardQuery.WILDCARD_ESCAPE).appendCodePoint(valuePoints[2]).toString()));+ query =+ new WildcardQuery(+ new Term(+ field,+ new StringBuilder()+ .append(WildcardQuery.WILDCARD_ESCAPE)+ .appendCodePoint(valuePoints[0])+ .append(WildcardQuery.WILDCARD_CHAR)+ .append(WildcardQuery.WILDCARD_ESCAPE)+ .appendCodePoint(valuePoints[2])+ .toString()));highlightAndAssertMatch(searcher, highlighter, query, field, value);- //TODO hmmm; how to randomly generate RegexpQuery? Low priority; we've covered the others well.+ // TODO hmmm; how to randomly generate RegexpQuery? Low priority; we've covered the others+ // well.}}}}- private void highlightAndAssertMatch(IndexSearcher searcher, UnifiedHighlighter highlighter, Query query, String field, String fieldVal) throws IOException {+ private void highlightAndAssertMatch(+ IndexSearcher searcher,+ UnifiedHighlighter highlighter,+ Query query,+ String field,+ String fieldVal)+ throws IOException {TopDocs topDocs = searcher.search(query, 1);assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighter.highlight(field, query, topDocs);- assertEquals("[<b>"+fieldVal+"</b>]", Arrays.toString(snippets));+ assertEquals("[<b>" + fieldVal + "</b>]", Arrays.toString(snippets));}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterRanking.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterRanking.javaindex 75758a32a02..cd85c6c9a9c 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterRanking.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterRanking.java@@ -21,7 +21,6 @@ import java.util.EnumSet;import java.util.HashSet;import java.util.Random;import java.util.Set;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -53,8 +52,8 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {final FieldType fieldType = UHTestHelper.randomFieldType(random());/**- * indexes a bunch of gibberish, and then highlights top(n).- * asserts that top(n) highlights is a subset of top(n+1) up to some max N+ * indexes a bunch of gibberish, and then highlights top(n). asserts that top(n) highlights is a+ * subset of top(n+1) up to some max N*/// TODO: this only tests single-valued fields. we should also index multiple values per field!public void testRanking() throws Exception {@@ -104,10 +103,11 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {checkQuery(is, new TermQuery(term), doc, maxTopN);// check a boolean queryTerm nextTerm = new Term("body", "" + (char) (ch + 1));- BooleanQuery bq = new BooleanQuery.Builder()- .add(new TermQuery(term), BooleanClause.Occur.SHOULD)- .add(new TermQuery(nextTerm), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery bq =+ new BooleanQuery.Builder()+ .add(new TermQuery(term), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(nextTerm), BooleanClause.Occur.SHOULD)+ .build();checkQuery(is, bq, doc, maxTopN);}}@@ -115,28 +115,31 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {private void checkQuery(IndexSearcher is, Query query, int doc, int maxTopN) throws IOException {for (int n = 1; n < maxTopN; n++) {final FakePassageFormatter f1 = new FakePassageFormatter();- UnifiedHighlighter p1 = new UnifiedHighlighter(is, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- assertEquals("body", field);- return f1;- }- };+ UnifiedHighlighter p1 =+ new UnifiedHighlighter(is, indexAnalyzer) {+ @Override+ protected PassageFormatter getFormatter(String field) {+ assertEquals("body", field);+ return f1;+ }+ };p1.setMaxLength(Integer.MAX_VALUE - 1);final FakePassageFormatter f2 = new FakePassageFormatter();- UnifiedHighlighter p2 = new UnifiedHighlighter(is, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- assertEquals("body", field);- return f2;- }- };+ UnifiedHighlighter p2 =+ new UnifiedHighlighter(is, indexAnalyzer) {+ @Override+ protected PassageFormatter getFormatter(String field) {+ assertEquals("body", field);+ return f2;+ }+ };p2.setMaxLength(Integer.MAX_VALUE - 1);BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();queryBuilder.add(query, BooleanClause.Occur.MUST);- queryBuilder.add(new TermQuery(new Term("id", Integer.toString(doc))), BooleanClause.Occur.MUST);+ queryBuilder.add(+ new TermQuery(new Term("id", Integer.toString(doc))), BooleanClause.Occur.MUST);BooleanQuery bq = queryBuilder.build();TopDocs td = is.search(bq, 1);p1.highlight("body", bq, td, n);@@ -146,8 +149,8 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {}/**- * returns a new random sentence, up to maxSentenceLength "words" in length.- * each word is a single character (a-z). The first one is capitalized.+ * returns a new random sentence, up to maxSentenceLength "words" in length. each word is a single+ * character (a-z). The first one is capitalized.*/private String newSentence(Random r, int maxSentenceLength) {StringBuilder sb = new StringBuilder();@@ -166,8 +169,8 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {}/**- * a fake formatter that doesn't actually format passages.- * instead it just collects them for asserts!+ * a fake formatter that doesn't actually format passages. instead it just collects them for+ * asserts!*/static class FakePassageFormatter extends PassageFormatter {HashSet<Pair> seen = new HashSet<>();@@ -199,7 +202,8 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {assertEquals(matchStart + 1, matchEnd);// and the offsets must be correct...assertEquals(1, term.length);- assertEquals((char) term.bytes[term.offset], Character.toLowerCase(content.charAt(matchStart)));+ assertEquals(+ (char) term.bytes[term.offset], Character.toLowerCase(content.charAt(matchStart)));}// record just the start/end offset for simplicityseen.add(new Pair(p.getStartOffset(), p.getEndOffset()));@@ -253,9 +257,7 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {}}- /**- * sets b=0 to disable passage length normalization- */+ /** sets b=0 to disable passage length normalization */public void testCustomB() throws Exception {Directory dir = newDirectory();indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);@@ -267,29 +269,34 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. This test is a better test but the sentence is excruiatingly long, " +- "you have no idea how painful it was for me to type this long sentence into my IDE.");+ body.setStringValue(+ "This is a test. This test is a better test but the sentence is excruiatingly long, "+ + "you have no idea how painful it was for me to type this long sentence into my IDE.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Set<HighlightFlag> getFlags(String field) {- if (random().nextBoolean()) {- return EnumSet.of(HighlightFlag.MULTI_TERM_QUERY, HighlightFlag.PHRASES, HighlightFlag.WEIGHT_MATCHES);- } else {- return super.getFlags(field);- }- }-- @Override- protected PassageScorer getScorer(String field) {- return new PassageScorer(1.2f, 0, 87);- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ if (random().nextBoolean()) {+ return EnumSet.of(+ HighlightFlag.MULTI_TERM_QUERY,+ HighlightFlag.PHRASES,+ HighlightFlag.WEIGHT_MATCHES);+ } else {+ return super.getFlags(field);+ }+ }++ @Override+ protected PassageScorer getScorer(String field) {+ return new PassageScorer(1.2f, 0, 87);+ }+ };Query query = new TermQuery(new Term("body", "test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);@@ -301,9 +308,7 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {dir.close();}- /**- * sets k1=0 for simple coordinate-level match (# of query terms present)- */+ /** sets k1=0 for simple coordinate-level match (# of query terms present) */public void testCustomK1() throws Exception {Directory dir = newDirectory();indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);@@ -315,34 +320,40 @@ public class TestUnifiedHighlighterRanking extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This has only foo foo. " +- "On the other hand this sentence contains both foo and bar. " +- "This has only bar bar bar bar bar bar bar bar bar bar bar bar.");+ body.setStringValue(+ "This has only foo foo. "+ + "On the other hand this sentence contains both foo and bar. "+ + "This has only bar bar bar bar bar bar bar bar bar bar bar bar.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Set<HighlightFlag> getFlags(String field) {- if (random().nextBoolean()) {- return EnumSet.of(HighlightFlag.MULTI_TERM_QUERY, HighlightFlag.PHRASES, HighlightFlag.WEIGHT_MATCHES);- } else {- return super.getFlags(field);- }- }-- @Override- protected PassageScorer getScorer(String field) {- return new PassageScorer(0, 0.75f, 87);- }- };- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "foo")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "bar")), BooleanClause.Occur.SHOULD)- .build();+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ if (random().nextBoolean()) {+ return EnumSet.of(+ HighlightFlag.MULTI_TERM_QUERY,+ HighlightFlag.PHRASES,+ HighlightFlag.WEIGHT_MATCHES);+ } else {+ return super.getFlags(field);+ }+ }++ @Override+ protected PassageScorer getScorer(String field) {+ return new PassageScorer(0, 0.75f, 87);+ }+ };+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "foo")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "bar")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 1);diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterReanalysis.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterReanalysis.javaindex 04ab01140c0..328caa43f6b 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterReanalysis.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterReanalysis.java@@ -17,7 +17,6 @@package org.apache.lucene.search.uhighlight;import java.io.IOException;-import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;import org.apache.lucene.index.IndexReader;@@ -35,38 +34,41 @@ import org.junit.Test;public class TestUnifiedHighlighterReanalysis extends LuceneTestCase {private MockAnalyzer indexAnalyzer =- new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase;+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercase;@Testpublic void testWithoutIndexSearcher() throws IOException {- String text = "This is a test. Just a test highlighting without a searcher. Feel free to ignore.";- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD)- .build();+ String text =+ "This is a test. Just a test highlighting without a searcher. Feel free to ignore.";+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "highlighting")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("title", "test")), BooleanClause.Occur.SHOULD)+ .build();UnifiedHighlighter highlighter = new UnifiedHighlighter(null, indexAnalyzer);String snippet = highlighter.highlightWithoutSearcher("body", query, text, 1).toString();assertEquals("Just a test <b>highlighting</b> without a searcher. ", snippet);- assertEquals("test single space", " ", highlighter.highlightWithoutSearcher("body", query, " ", 1));+ assertEquals(+ "test single space", " ", highlighter.highlightWithoutSearcher("body", query, " ", 1));assertEquals("Hello", highlighter.highlightWithoutSearcher("nonexistent", query, "Hello", 1));}@Test(expected = IllegalStateException.class)public void testIndexSearcherNullness() throws IOException {- String text = "This is a test. Just a test highlighting without a searcher. Feel free to ignore.";+ String text =+ "This is a test. Just a test highlighting without a searcher. Feel free to ignore.";Query query = new TermQuery(new Term("body", "highlighting"));try (Directory directory = newDirectory();- RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);- IndexReader indexReader = indexWriter.getReader()) {+ RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);+ IndexReader indexReader = indexWriter.getReader()) {IndexSearcher searcher = newSearcher(indexReader);UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);- highlighter.highlightWithoutSearcher("body", query, text, 1);//should throw+ highlighter.highlightWithoutSearcher("body", query, text, 1); // should throw}}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.javaindex 4c05e1b9f66..a5a4ffdf9aa 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterStrictPhrases.java@@ -16,6 +16,7 @@*/package org.apache.lucene.search.uhighlight;+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.IOException;import java.util.Collection;import java.util.Collections;@@ -23,8 +24,6 @@ import java.util.EnumSet;import java.util.Locale;import java.util.Set;import java.util.concurrent.atomic.AtomicBoolean;--import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;import org.apache.lucene.document.Document;@@ -64,7 +63,7 @@ import org.apache.lucene.util.QueryBuilder;import org.junit.After;import org.junit.Before;-//TODO rename to reflect position sensitivity+// TODO rename to reflect position sensitivitypublic class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {final FieldType fieldType;@@ -90,8 +89,10 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {@Beforepublic void doBefore() throws IOException {- indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase- indexAnalyzer.setPositionIncrementGap(3);// more than default+ indexAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercase+ indexAnalyzer.setPositionIncrementGap(3); // more than defaultdir = newDirectory();indexWriter = new RandomIndexWriter(random(), dir, indexAnalyzer);}@@ -112,32 +113,41 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {private void initReaderSearcherHighlighter() throws IOException {indexReader = indexWriter.getReader();searcher = newSearcher(indexReader);- highlighter = TestUnifiedHighlighter.randomUnifiedHighlighter(searcher, indexAnalyzer,- EnumSet.of(HighlightFlag.PHRASES, HighlightFlag.MULTI_TERM_QUERY), true);+ highlighter =+ TestUnifiedHighlighter.randomUnifiedHighlighter(+ searcher,+ indexAnalyzer,+ EnumSet.of(HighlightFlag.PHRASES, HighlightFlag.MULTI_TERM_QUERY),+ true);// intercept the formatter in order to check constraints on the passage.final PassageFormatter defaultFormatter = highlighter.getFormatter(null);- highlighter.setFormatter(new PassageFormatter() {- @Override- public Object format(Passage[] passages, String content) {- boolean thisDupMatchAllowed = dupMatchAllowed.getAndSet(true);- for (Passage passage : passages) {- String prevPair = "";- for (int i = 0; i < passage.getNumMatches(); i++) {- // pad each to make comparable- String pair = String.format(Locale.ROOT, "%03d-%03d", passage.getMatchStarts()[i], passage.getMatchEnds()[i]);- int cmp = prevPair.compareTo(pair);- if (cmp == 0) {- assertTrue("dup match in passage at offset " + pair, thisDupMatchAllowed);- } else if (cmp > 0) {- fail("bad match order in passage at offset " + pair);+ highlighter.setFormatter(+ new PassageFormatter() {+ @Override+ public Object format(Passage[] passages, String content) {+ boolean thisDupMatchAllowed = dupMatchAllowed.getAndSet(true);+ for (Passage passage : passages) {+ String prevPair = "";+ for (int i = 0; i < passage.getNumMatches(); i++) {+ // pad each to make comparable+ String pair =+ String.format(+ Locale.ROOT,+ "%03d-%03d",+ passage.getMatchStarts()[i],+ passage.getMatchEnds()[i]);+ int cmp = prevPair.compareTo(pair);+ if (cmp == 0) {+ assertTrue("dup match in passage at offset " + pair, thisDupMatchAllowed);+ } else if (cmp > 0) {+ fail("bad match order in passage at offset " + pair);+ }+ prevPair = pair;+ }}- prevPair = pair;+ return defaultFormatter.format(passages, content);}-- }- return defaultFormatter.format(passages, content);- }- });+ });}private PhraseQuery newPhraseQuery(String field, String phrase) {@@ -156,23 +166,24 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {}public void testBasics() throws IOException {- indexWriter.addDocument(newDoc("Yin yang, filter")); // filter out. test getTermToSpanLists reader 1-doc filter+ indexWriter.addDocument(+ newDoc("Yin yang, filter")); // filter out. test getTermToSpanLists reader 1-doc filterindexWriter.addDocument(newDoc("yin alone, Yin yang, yin gap yang"));initReaderSearcherHighlighter();- //query: -filter +"yin yang"- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "filter")), BooleanClause.Occur.MUST_NOT)- .add(newPhraseQuery("body", "yin yang"), BooleanClause.Occur.MUST)- .build();-+ // query: -filter +"yin yang"+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "filter")), BooleanClause.Occur.MUST_NOT)+ .add(newPhraseQuery("body", "yin yang"), BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"yin alone, <b>Yin yang</b>, yin gap yang"}, snippets);+ assertArrayEquals(new String[] {"yin alone, <b>Yin yang</b>, yin gap yang"}, snippets);} else {- assertArrayEquals(new String[]{"yin alone, <b>Yin</b> <b>yang</b>, yin gap yang"}, snippets);+ assertArrayEquals(new String[] {"yin alone, <b>Yin</b> <b>yang</b>, yin gap yang"}, snippets);}}@@ -180,59 +191,67 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {indexWriter.addDocument(newDoc("Yin yang, yin gap yang"));initReaderSearcherHighlighter();- BooleanQuery query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "yin")), BooleanClause.Occur.MUST)- .add(newPhraseQuery("body", "yin yang"), BooleanClause.Occur.MUST)- // add queries for other fields; we shouldn't highlight these because of that.- .add(new TermQuery(new Term("title", "yang")), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "yin")), BooleanClause.Occur.MUST)+ .add(newPhraseQuery("body", "yin yang"), BooleanClause.Occur.MUST)+ // add queries for other fields; we shouldn't highlight these because of that.+ .add(new TermQuery(new Term("title", "yang")), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);- dupMatchAllowed.set(false); // We don't want duplicates from "Yin" being in TermQuery & PhraseQuery.+ dupMatchAllowed.set(+ false); // We don't want duplicates from "Yin" being in TermQuery & PhraseQuery.String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>Yin yang</b>, <b>yin</b> gap yang"}, snippets);+ assertArrayEquals(new String[] {"<b>Yin yang</b>, <b>yin</b> gap yang"}, snippets);} else {- assertArrayEquals(new String[]{"<b>Yin</b> <b>yang</b>, <b>yin</b> gap yang"}, snippets);+ assertArrayEquals(new String[] {"<b>Yin</b> <b>yang</b>, <b>yin</b> gap yang"}, snippets);}}public void testPhraseNotInDoc() throws IOException {indexWriter.addDocument(newDoc("Whatever yin")); // query matches this; highlight it- indexWriter.addDocument(newDoc("nextdoc yin"));// query does NOT match this, only the SHOULD clause does+ indexWriter.addDocument(+ newDoc("nextdoc yin")); // query does NOT match this, only the SHOULD clause doesinitReaderSearcherHighlighter();- BooleanQuery query = new BooleanQuery.Builder()- //MUST:- .add(new TermQuery(new Term("body", "whatever")), BooleanClause.Occur.MUST)- //SHOULD: (yet won't)- .add(newPhraseQuery("body", "nextdoc yin"), BooleanClause.Occur.SHOULD)- .add(newPhraseQuery("body", "nonexistent yin"), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ // MUST:+ .add(new TermQuery(new Term("body", "whatever")), BooleanClause.Occur.MUST)+ // SHOULD: (yet won't)+ .add(newPhraseQuery("body", "nextdoc yin"), BooleanClause.Occur.SHOULD)+ .add(newPhraseQuery("body", "nonexistent yin"), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);- assertArrayEquals(new String[]{"<b>Whatever</b> yin"}, snippets);+ assertArrayEquals(new String[] {"<b>Whatever</b> yin"}, snippets);}public void testSubPhrases() throws IOException {indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));initReaderSearcherHighlighter();- BooleanQuery query = new BooleanQuery.Builder()- .add(newPhraseQuery("body", "alpha bravo charlie"), BooleanClause.Occur.MUST)- .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(newPhraseQuery("body", "alpha bravo charlie"), BooleanClause.Occur.MUST)+ .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);dupMatchAllowed.set(false); // We don't want duplicates from both PhraseQueryString[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>alpha bravo charlie</b> - charlie bravo alpha"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha bravo charlie</b> - charlie bravo alpha"}, snippets);} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"},+ snippets);}}@@ -240,161 +259,185 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {indexWriter.addDocument(newDoc("mother father w mom father w dad"));initReaderSearcherHighlighter();- MultiPhraseQuery query = new MultiPhraseQuery.Builder()- .add(new Term[]{new Term("body", "mom"), new Term("body", "mother")})- .add(new Term[]{new Term("body", "dad"), new Term("body", "father")})- .build();+ MultiPhraseQuery query =+ new MultiPhraseQuery.Builder()+ .add(new Term[] {new Term("body", "mom"), new Term("body", "mother")})+ .add(new Term[] {new Term("body", "dad"), new Term("body", "father")})+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>mother father</b> w <b>mom father</b> w dad"}, snippets);+ assertArrayEquals(new String[] {"<b>mother father</b> w <b>mom father</b> w dad"}, snippets);} else {- assertArrayEquals(new String[]{"<b>mother</b> <b>father</b> w <b>mom</b> <b>father</b> w dad"}, snippets);+ assertArrayEquals(+ new String[] {"<b>mother</b> <b>father</b> w <b>mom</b> <b>father</b> w dad"}, snippets);}}/**- * Test it does *not* highlight the same term's not next to the span-near. "charlie" in this case.+ * Test it does *not* highlight the same term's not next to the span-near. "charlie" in this case.* This particular example exercises "Rewrite" plus "MTQ" in the same query.*/public void testRewriteAndMtq() throws IOException {indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));initReaderSearcherHighlighter();- SpanNearQuery snq = new SpanNearQuery(- new SpanQuery[]{- new SpanTermQuery(new Term("body", "bravo")),- new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term("body", "ch")))}, // REWRITES- 0, true);-- BooleanQuery query = new BooleanQuery.Builder()- .add(snq, BooleanClause.Occur.MUST)- .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ- .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)- // add queries for other fields; we shouldn't highlight these because of that.- .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)- .build();+ SpanNearQuery snq =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("body", "bravo")),+ new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term("body", "ch")))+ }, // REWRITES+ 0,+ true);++ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(snq, BooleanClause.Occur.MUST)+ .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ+ .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)+ // add queries for other fields; we shouldn't highlight these because of that.+ .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"},+ snippets);} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"},+ snippets);}// do again, this time with MTQ disabled. We should only find "alpha bravo".highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setHandleMultiTermQuery(false);//disable but leave phrase processing enabled+ highlighter.setHandleMultiTermQuery(false); // disable but leave phrase processing enabledtopDocs = searcher.search(query, 10, Sort.INDEXORDER);snippets = highlighter.highlight("body", query, topDocs);- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> charlie - charlie bravo alpha"},- snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> charlie - charlie bravo alpha"}, snippets);}- /**- * Like {@link #testRewriteAndMtq} but no freestanding MTQ- */+ /** Like {@link #testRewriteAndMtq} but no freestanding MTQ */public void testRewrite() throws IOException {indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));initReaderSearcherHighlighter();- SpanNearQuery snq = new SpanNearQuery(- new SpanQuery[]{- new SpanTermQuery(new Term("body", "bravo")),- new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term("body", "ch")))}, // REWRITES- 0, true);- BooleanQuery query = new BooleanQuery.Builder()- .add(snq, BooleanClause.Occur.MUST)-// .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ- .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)- // add queries for other fields; we shouldn't highlight these because of that.- .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)- .build();+ SpanNearQuery snq =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("body", "bravo")),+ new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term("body", "ch")))+ }, // REWRITES+ 0,+ true);+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(snq, BooleanClause.Occur.MUST)+ // .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) //+ // MTQ+ .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)+ // add queries for other fields; we shouldn't highlight these because of that.+ .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>alpha bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"},+ snippets);}// do again, this time with MTQ disabled. We should only find "alpha bravo".highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setHandleMultiTermQuery(false);//disable but leave phrase processing enabled+ highlighter.setHandleMultiTermQuery(false); // disable but leave phrase processing enabledtopDocs = searcher.search(query, 10, Sort.INDEXORDER);snippets = highlighter.highlight("body", query, topDocs);- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> charlie - charlie bravo alpha"},- snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> charlie - charlie bravo alpha"}, snippets);}- /**- * Like {@link #testRewriteAndMtq} but no rewrite.- */+ /** Like {@link #testRewriteAndMtq} but no rewrite. */public void testMtq() throws IOException {indexWriter.addDocument(newDoc("alpha bravo charlie - charlie bravo alpha"));initReaderSearcherHighlighter();- SpanNearQuery snq = new SpanNearQuery(- new SpanQuery[]{- new SpanTermQuery(new Term("body", "bravo")),- new SpanTermQuery(new Term("body", "charlie"))}, // does NOT rewrite- 0, true);-- BooleanQuery query = new BooleanQuery.Builder()- .add(snq, BooleanClause.Occur.MUST)- .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ- .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)- // add queries for other fields; we shouldn't highlight these because of that.- .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)- .build();+ SpanNearQuery snq =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("body", "bravo")),+ new SpanTermQuery(new Term("body", "charlie"))+ }, // does NOT rewrite+ 0,+ true);++ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(snq, BooleanClause.Occur.MUST)+ .add(new PrefixQuery(new Term("body", "al")), BooleanClause.Occur.MUST) // MTQ+ .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.MUST)+ // add queries for other fields; we shouldn't highlight these because of that.+ .add(newPhraseQuery("title", "bravo alpha"), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"},+ snippets);} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"}, snippets);+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo <b>alpha</b>"},+ snippets);}// do again, this time with MTQ disabled.highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setHandleMultiTermQuery(false);//disable but leave phrase processing enabled+ highlighter.setHandleMultiTermQuery(false); // disable but leave phrase processing enabledtopDocs = searcher.search(query, 10, Sort.INDEXORDER);snippets = highlighter.highlight("body", query, topDocs);- //note: without MTQ, the WEIGHT_MATCHES is disabled which affects the snippet boundaries- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"},- snippets);+ // note: without MTQ, the WEIGHT_MATCHES is disabled which affects the snippet boundaries+ assertArrayEquals(+ new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> - charlie bravo alpha"}, snippets);}public void testMultiValued() throws IOException {indexWriter.addDocument(newDoc("one bravo three", "four bravo six"));initReaderSearcherHighlighter();- BooleanQuery query = new BooleanQuery.Builder()- .add(newPhraseQuery("body", "one bravo"), BooleanClause.Occur.MUST)- .add(newPhraseQuery("body", "four bravo"), BooleanClause.Occur.MUST)- .add(new PrefixQuery(new Term("body", "br")), BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(newPhraseQuery("body", "one bravo"), BooleanClause.Occur.MUST)+ .add(newPhraseQuery("body", "four bravo"), BooleanClause.Occur.MUST)+ .add(new PrefixQuery(new Term("body", "br")), BooleanClause.Occur.MUST)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs, 2);if (highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertArrayEquals(new String[]{"<b>one bravo</b> three... <b>four bravo</b> six"}, snippets);+ assertArrayEquals(new String[] {"<b>one bravo</b> three... <b>four bravo</b> six"}, snippets);} else {- assertArrayEquals(new String[]{"<b>one</b> <b>bravo</b> three... <b>four</b> <b>bravo</b> six"}, snippets);+ assertArrayEquals(+ new String[] {"<b>one</b> <b>bravo</b> three... <b>four</b> <b>bravo</b> six"}, snippets);}// now test phraseQuery won't span across values@@ -404,10 +447,11 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {// 1 too little; won't spanphraseQuery = setSlop(phraseQuery, indexAnalyzer.getPositionIncrementGap("body") - 1);- query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "bravo")), BooleanClause.Occur.MUST)- .add(phraseQuery, BooleanClause.Occur.SHOULD)- .build();+ query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "bravo")), BooleanClause.Occur.MUST)+ .add(phraseQuery, BooleanClause.Occur.SHOULD)+ .build();topDocs = searcher.search(query, 10);snippets = highlighter.highlight("body", query, topDocs, 2);@@ -415,11 +459,13 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {// and add just enough slop to cross the values:phraseQuery = newPhraseQuery("body", "three four");- phraseQuery = setSlop(phraseQuery, indexAnalyzer.getPositionIncrementGap("body")); // just enough to span- query = new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "bravo")), BooleanClause.Occur.MUST)- .add(phraseQuery, BooleanClause.Occur.MUST) // must match and it will- .build();+ phraseQuery =+ setSlop(phraseQuery, indexAnalyzer.getPositionIncrementGap("body")); // just enough to span+ query =+ new BooleanQuery.Builder()+ .add(new TermQuery(new Term("body", "bravo")), BooleanClause.Occur.MUST)+ .add(phraseQuery, BooleanClause.Occur.MUST) // must match and it will+ .build();topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);snippets = highlighter.highlight("body", query, topDocs, 2);@@ -431,28 +477,31 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {}public void testMaxLen() throws IOException {- indexWriter.addDocument(newDoc("alpha bravo charlie - gap alpha bravo")); // hyphen is at char 21+ indexWriter.addDocument(+ newDoc("alpha bravo charlie - gap alpha bravo")); // hyphen is at char 21initReaderSearcherHighlighter();highlighter.setMaxLength(21);- BooleanQuery query = new BooleanQuery.Builder()- .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.SHOULD)- .add(newPhraseQuery("body", "gap alpha"), BooleanClause.Occur.SHOULD)- .add(newPhraseQuery("body", "charlie gap"), BooleanClause.Occur.SHOULD)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder()+ .add(newPhraseQuery("body", "alpha bravo"), BooleanClause.Occur.SHOULD)+ .add(newPhraseQuery("body", "gap alpha"), BooleanClause.Occur.SHOULD)+ .add(newPhraseQuery("body", "charlie gap"), BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);- final boolean weightMatches = highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES);+ final boolean weightMatches =+ highlighter.getFlags("body").contains(HighlightFlag.WEIGHT_MATCHES);if (fieldType == UHTestHelper.reanalysisType || weightMatches) {if (weightMatches) {- assertArrayEquals(new String[]{"<b>alpha bravo</b> charlie -"}, snippets);+ assertArrayEquals(new String[] {"<b>alpha bravo</b> charlie -"}, snippets);} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> charlie -"}, snippets);+ assertArrayEquals(new String[] {"<b>alpha</b> <b>bravo</b> charlie -"}, snippets);}} else {- assertArrayEquals(new String[]{"<b>alpha</b> <b>bravo</b> <b>charlie</b> -"}, snippets);+ assertArrayEquals(new String[] {"<b>alpha</b> <b>bravo</b> <b>charlie</b> -"}, snippets);}}@@ -461,18 +510,20 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {initReaderSearcherHighlighter();WildcardQuery wildcardQuery = new WildcardQuery(new Term("body", "free*"));- SpanMultiTermQueryWrapper<WildcardQuery> wildcardSpanQuery = new SpanMultiTermQueryWrapper<>(wildcardQuery);+ SpanMultiTermQueryWrapper<WildcardQuery> wildcardSpanQuery =+ new SpanMultiTermQueryWrapper<>(wildcardQuery);SpanTermQuery termQuery = new SpanTermQuery(new Term("body", "speech"));- SpanQuery spanQuery = new SpanNearQuery(new SpanQuery[]{wildcardSpanQuery, termQuery}, 3, false);+ SpanQuery spanQuery =+ new SpanNearQuery(new SpanQuery[] {wildcardSpanQuery, termQuery}, 3, false);- BooleanQuery query = new BooleanQuery.Builder()- .add(spanQuery, BooleanClause.Occur.MUST)- .build();+ BooleanQuery query =+ new BooleanQuery.Builder().add(spanQuery, BooleanClause.Occur.MUST).build();TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);// spans' MatchesIterator exposes each underlying term; thus doesn't enclose intermediate "of"- assertArrayEquals(new String[]{"freezing cold stuff like stuff <b>freedom</b> of <b>speech</b>"}, snippets);+ assertArrayEquals(+ new String[] {"freezing cold stuff like stuff <b>freedom</b> of <b>speech</b>"}, snippets);}public void testMatchNoDocsQuery() throws IOException {@@ -484,47 +535,58 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {}public void testPreSpanQueryRewrite() throws IOException {- indexWriter.addDocument(newDoc("There is no accord and satisfaction with this - Consideration of the accord is arbitrary."));+ indexWriter.addDocument(+ newDoc(+ "There is no accord and satisfaction with this - Consideration of the accord is arbitrary."));initReaderSearcherHighlighter();- highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Set<HighlightFlag> getFlags(String field) {- final Set<HighlightFlag> flags = super.getFlags(field);- flags.remove(HighlightFlag.WEIGHT_MATCHES);//unsupported- return flags;- }+ highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ final Set<HighlightFlag> flags = super.getFlags(field);+ flags.remove(HighlightFlag.WEIGHT_MATCHES); // unsupported+ return flags;+ }- @Override- protected Collection<Query> preSpanQueryRewrite(Query query) {- if (query instanceof MyQuery) {- return Collections.singletonList(((MyQuery)query).wrapped);- }- return null;- }- };+ @Override+ protected Collection<Query> preSpanQueryRewrite(Query query) {+ if (query instanceof MyQuery) {+ return Collections.singletonList(((MyQuery) query).wrapped);+ }+ return null;+ }+ };highlighter.setHighlightPhrasesStrictly(true);BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();- Query phraseQuery = new BoostQuery(new PhraseQuery("body", "accord", "and", "satisfaction"), 2.0f);- Query oredTerms = new BooleanQuery.Builder()- .setMinimumNumberShouldMatch(2)- .add(new TermQuery(new Term("body", "accord")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "satisfaction")), BooleanClause.Occur.SHOULD)- .add(new TermQuery(new Term("body", "consideration")), BooleanClause.Occur.SHOULD)- .build();+ Query phraseQuery =+ new BoostQuery(new PhraseQuery("body", "accord", "and", "satisfaction"), 2.0f);+ Query oredTerms =+ new BooleanQuery.Builder()+ .setMinimumNumberShouldMatch(2)+ .add(new TermQuery(new Term("body", "accord")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "satisfaction")), BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("body", "consideration")), BooleanClause.Occur.SHOULD)+ .build();Query proximityBoostingQuery = new MyQuery(oredTerms);- Query totalQuery = bqBuilder- .add(phraseQuery, BooleanClause.Occur.SHOULD)- .add(proximityBoostingQuery, BooleanClause.Occur.SHOULD)- .build();+ Query totalQuery =+ bqBuilder+ .add(phraseQuery, BooleanClause.Occur.SHOULD)+ .add(proximityBoostingQuery, BooleanClause.Occur.SHOULD)+ .build();TopDocs topDocs = searcher.search(totalQuery, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighter.highlight("body", totalQuery, topDocs);- assertArrayEquals(new String[]{"There is no <b>accord</b> <b>and</b> <b>satisfaction</b> with this - <b>Consideration</b> of the <b>accord</b> is arbitrary."}, snippets);+ assertArrayEquals(+ new String[] {+ "There is no <b>accord</b> <b>and</b> <b>satisfaction</b> with this - <b>Consideration</b> of the <b>accord</b> is arbitrary."+ },+ snippets);}- // Tests that terms collected out of order due to being present in multiple Spans are handled correctly+ // Tests that terms collected out of order due to being present in multiple Spans are handled+ // correctly// See LUCENE-8365public void testReverseOrderSpanCollection() throws IOException {// Processing order may depend on various optimizations or other weird factor.@@ -532,23 +594,34 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {indexWriter.addDocument(newDoc("alpha charlie - alpha bravo"));initReaderSearcherHighlighter();- SpanNearQuery query = new SpanNearQuery(new SpanQuery[]{- new SpanNearQuery(new SpanQuery[]{- new SpanTermQuery(new Term("body", "alpha")),- new SpanTermQuery(new Term("body", "bravo"))- }, 0, true),- new SpanNearQuery(new SpanQuery[]{- new SpanTermQuery(new Term("body", "alpha")),- new SpanTermQuery(new Term("body", "charlie"))- }, 0, true)- }, 10, false);+ SpanNearQuery query =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("body", "alpha")),+ new SpanTermQuery(new Term("body", "bravo"))+ },+ 0,+ true),+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term("body", "alpha")),+ new SpanTermQuery(new Term("body", "charlie"))+ },+ 0,+ true)+ },+ 10,+ false);TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);String[] snippets = highlighter.highlight("body", query, topDocs);- assertArrayEquals(new String[]{- "<b>alpha</b> <b>bravo</b> - <b>alpha</b> <b>charlie</b>",- "<b>alpha</b> <b>charlie</b> - <b>alpha</b> <b>bravo</b>",+ assertArrayEquals(+ new String[] {+ "<b>alpha</b> <b>bravo</b> - <b>alpha</b> <b>charlie</b>",+ "<b>alpha</b> <b>charlie</b> - <b>alpha</b> <b>bravo</b>",},snippets);}@@ -562,7 +635,8 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {}@Override- public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)+ throws IOException {return wrapped.createWeight(searcher, scoreMode, boost);}@@ -577,12 +651,14 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {@Overridepublic String toString(String field) {- return "[[["+wrapped.toString(field)+"]]]";+ return "[[[" + wrapped.toString(field) + "]]]";}@Overridepublic boolean equals(Object obj) {- return obj != null && obj.getClass() == getClass() && wrapped.equals(((MyQuery)wrapped).wrapped);+ return obj != null+ && obj.getClass() == getClass()+ && wrapped.equals(((MyQuery) wrapped).wrapped);}@Override@@ -598,64 +674,115 @@ public class TestUnifiedHighlighterStrictPhrases extends LuceneTestCase {// Ported from LUCENE-5455 (fixed in LUCENE-8121). Also see LUCENE-2287.public void testNestedSpanQueryHighlight() throws Exception {- // For a long time, the highlighters used to assume all query terms within the SpanQuery were valid at the Spans'- // position range. This would highlight occurrences of terms that were actually not matched by the query.+ // For a long time, the highlighters used to assume all query terms within the SpanQuery were+ // valid at the Spans'+ // position range. This would highlight occurrences of terms that were actually not matched+ // by the query.// But now using the SpanCollector API we don't make this kind of mistake.final String FIELD_NAME = "body";final String indexedText = "x y z x z x a";indexWriter.addDocument(newDoc(indexedText));initReaderSearcherHighlighter();- TopDocs topDocs = new TopDocs(new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[]{new ScoreDoc(0, 1f)});+ TopDocs topDocs =+ new TopDocs(+ new TotalHits(1, TotalHits.Relation.EQUAL_TO), new ScoreDoc[] {new ScoreDoc(0, 1f)});String expected = "<b>x</b> <b>y</b> <b>z</b> x z x <b>a</b>";- Query q = new SpanNearQuery(new SpanQuery[] {- new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "x")),- new SpanTermQuery(new Term(FIELD_NAME, "y")),- new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),- new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);+ Query q =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "x")),+ new SpanTermQuery(new Term(FIELD_NAME, "y")),+ new SpanTermQuery(new Term(FIELD_NAME, "z"))+ },+ 0,+ true),+ new SpanTermQuery(new Term(FIELD_NAME, "a"))+ },+ 10,+ false);String observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);assertEquals("Nested SpanNear query not properly highlighted.", expected, observed);expected = "x <b>y</b> <b>z</b> <b>x</b> <b>z</b> x <b>a</b>";- q = new SpanNearQuery(new SpanQuery[] {- new SpanOrQuery(- new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "x")),- new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),- new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "y")),- new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true)),- new SpanOrQuery(- new SpanTermQuery(new Term(FIELD_NAME, "a")),- new SpanTermQuery(new Term(FIELD_NAME, "b")))}, 10, false);+ q =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanOrQuery(+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "x")),+ new SpanTermQuery(new Term(FIELD_NAME, "z"))+ },+ 0,+ true),+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "y")),+ new SpanTermQuery(new Term(FIELD_NAME, "z"))+ },+ 0,+ true)),+ new SpanOrQuery(+ new SpanTermQuery(new Term(FIELD_NAME, "a")),+ new SpanTermQuery(new Term(FIELD_NAME, "b")))+ },+ 10,+ false);observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);- assertEquals("Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);+ assertEquals(+ "Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);expected = "x <b>y</b> <b>z</b> <b>x</b> <b>z</b> x <b>a</b>";- q = new SpanNearQuery(new SpanQuery[] {- new SpanNearQuery(new SpanQuery[] {- new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term(FIELD_NAME, "*"))),- new SpanTermQuery(new Term(FIELD_NAME, "z"))}, 0, true),- new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);+ q =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term(FIELD_NAME, "*"))),+ new SpanTermQuery(new Term(FIELD_NAME, "z"))+ },+ 0,+ true),+ new SpanTermQuery(new Term(FIELD_NAME, "a"))+ },+ 10,+ false);observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);- assertEquals("Nested SpanNear query with wildcard not properly highlighted.", expected, observed);+ assertEquals(+ "Nested SpanNear query with wildcard not properly highlighted.", expected, observed);expected = "<b>x</b> <b>y</b> z x z x <b>a</b>";- q = new SpanNearQuery(new SpanQuery[] {- new SpanOrQuery(- new SpanNearQuery(new SpanQuery[] {- new SpanTermQuery(new Term(FIELD_NAME, "x")),- new SpanTermQuery(new Term(FIELD_NAME, "y"))}, 0, true),- new SpanNearQuery(new SpanQuery[] { //No hit span query- new SpanTermQuery(new Term(FIELD_NAME, "z")),- new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 0, true)),- new SpanTermQuery(new Term(FIELD_NAME, "a"))}, 10, false);+ q =+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanOrQuery(+ new SpanNearQuery(+ new SpanQuery[] {+ new SpanTermQuery(new Term(FIELD_NAME, "x")),+ new SpanTermQuery(new Term(FIELD_NAME, "y"))+ },+ 0,+ true),+ new SpanNearQuery(+ new SpanQuery[] { // No hit span query+ new SpanTermQuery(new Term(FIELD_NAME, "z")),+ new SpanTermQuery(new Term(FIELD_NAME, "a"))+ },+ 0,+ true)),+ new SpanTermQuery(new Term(FIELD_NAME, "a"))+ },+ 10,+ false);observed = highlighter.highlight(FIELD_NAME, q, topDocs)[0];if (VERBOSE) System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);- assertEquals("Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);+ assertEquals(+ "Nested SpanNear query within SpanOr not properly highlighted.", expected, observed);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermIntervals.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermIntervals.javaindex edf02121ffa..5817e01c59e 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermIntervals.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermIntervals.java@@ -16,6 +16,7 @@*/package org.apache.lucene.search.uhighlight;+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStreamReader;@@ -29,7 +30,6 @@ import java.util.Map;import java.util.Set;import java.util.regex.Matcher;import java.util.regex.Pattern;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -55,11 +55,10 @@ import org.apache.lucene.util.LuceneTestCase;import org.junit.After;import org.junit.Before;-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;-public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {- private final FieldType fieldType; // for "body" generally, but not necessarily others. See constructor+ private final FieldType+ fieldType; // for "body" generally, but not necessarily others. See constructorprivate MockAnalyzer indexAnalyzer;private Directory dir;@@ -75,7 +74,9 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {@Beforepublic void doBefore() throws IOException {- indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase+ indexAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercasedir = newDirectory();}@@ -84,44 +85,52 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {dir.close();}- static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer) {- return randomUnifiedHighlighter(searcher, indexAnalyzer, EnumSet.noneOf(HighlightFlag.class), null);- }-- static UnifiedHighlighter randomUnifiedHighlighter(IndexSearcher searcher, Analyzer indexAnalyzer,- EnumSet<HighlightFlag> mandatoryFlags, Boolean requireFieldMatch) {- final UnifiedHighlighter uh = new UnifiedHighlighter(searcher, indexAnalyzer) {- Set<HighlightFlag> flags; // consistently random set of flags for this test run- @Override- protected Set<HighlightFlag> getFlags(String field) {- if (flags != null) {- return flags;- }- final EnumSet<HighlightFlag> result = EnumSet.copyOf(mandatoryFlags);- int r = random().nextInt();- for (HighlightFlag highlightFlag : HighlightFlag.values()) {- if (((1 << highlightFlag.ordinal()) & r) == 0) {- result.add(highlightFlag);+ static UnifiedHighlighter randomUnifiedHighlighter(+ IndexSearcher searcher, Analyzer indexAnalyzer) {+ return randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.noneOf(HighlightFlag.class), null);+ }++ static UnifiedHighlighter randomUnifiedHighlighter(+ IndexSearcher searcher,+ Analyzer indexAnalyzer,+ EnumSet<HighlightFlag> mandatoryFlags,+ Boolean requireFieldMatch) {+ final UnifiedHighlighter uh =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ Set<HighlightFlag> flags; // consistently random set of flags for this test run++ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ if (flags != null) {+ return flags;+ }+ final EnumSet<HighlightFlag> result = EnumSet.copyOf(mandatoryFlags);+ int r = random().nextInt();+ for (HighlightFlag highlightFlag : HighlightFlag.values()) {+ if (((1 << highlightFlag.ordinal()) & r) == 0) {+ result.add(highlightFlag);+ }+ }+ if (result.contains(HighlightFlag.WEIGHT_MATCHES)) {+ // these two are required for WEIGHT_MATCHES+ result.add(HighlightFlag.MULTI_TERM_QUERY);+ result.add(HighlightFlag.PHRASES);+ }+ return flags = result;}- }- if (result.contains(HighlightFlag.WEIGHT_MATCHES)) {- // these two are required for WEIGHT_MATCHES- result.add(HighlightFlag.MULTI_TERM_QUERY);- result.add(HighlightFlag.PHRASES);- }- return flags = result;- }- };+ };uh.setCacheFieldValCharsThreshold(random().nextInt(100));- if (requireFieldMatch == Boolean.FALSE || (requireFieldMatch == null && random().nextBoolean())) {+ if (requireFieldMatch == Boolean.FALSE+ || (requireFieldMatch == null && random().nextBoolean())) {uh.setFieldMatcher(f -> true); // requireFieldMatch==false}return uh;-}//- // Tests below were ported from the PostingsHighlighter. Possibly augmented. Far below are newer tests.+ // Tests below were ported from the PostingsHighlighter. Possibly augmented. Far below are newer+ // tests.//public void testBasics() throws Exception {@@ -131,7 +140,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -303,7 +313,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -313,10 +324,11 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- Query query = new IntervalQuery("body", Intervals.or(- Intervals.term("highlighting"),- Intervals.term("just"),- Intervals.term("first")));+ Query query =+ new IntervalQuery(+ "body",+ Intervals.or(+ Intervals.term("highlighting"), Intervals.term("just"), Intervals.term("first")));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);@@ -333,7 +345,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("This test is another test. Not a good sentence. Test test test test.");iw.addDocument(doc);@@ -348,21 +361,25 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {assertEquals(2, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(2, snippets.length);- assertEquals("This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);- assertEquals("This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.", snippets[1]);+ assertEquals(+ "This is a <b>test</b>. Just a <b>test</b> highlighting from postings. ", snippets[0]);+ assertEquals(+ "This <b>test</b> is another <b>test</b>. ... <b>Test</b> <b>test</b> <b>test</b> <b>test</b>.",+ snippets[1]);ir.close();}public void testBuddhism() throws Exception {- String text = "This eight-volume set brings together seminal papers in Buddhist studies from a vast " +- "range of academic disciplines published over the last forty years. With a new introduction " +- "by the editor, this collection is a unique and unrivalled research resource for both " +- "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in " +- "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine " +- "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma " +- "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric " +- "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and " +- "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";+ String text =+ "This eight-volume set brings together seminal papers in Buddhist studies from a vast "+ + "range of academic disciplines published over the last forty years. With a new introduction "+ + "by the editor, this collection is a unique and unrivalled research resource for both "+ + "student and scholar. Coverage includes: - Buddhist origins; early history of Buddhism in "+ + "South and Southeast Asia - early Buddhist Schools and Doctrinal History; Theravada Doctrine "+ + "- the Origins and nature of Mahayana Buddhism; some Mahayana religious topics - Abhidharma "+ + "and Madhyamaka - Yogacara, the Epistemological tradition, and Tathagatagarbha - Tantric "+ + "Buddhism (Including China and Japan); Buddhism in Nepal and Tibet - Buddhism in South and "+ + "Southeast Asia, and - Buddhism in China, East Asia, and Japan.";RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Field body = new Field("body", text, fieldType);@@ -379,17 +396,19 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {highlighter.setHighlightPhrasesStrictly(false);String snippets[] = highlighter.highlight("body", query, topDocs, 2);assertEquals(1, snippets.length);- // highlighter.getFlags("body").containsAll(EnumSet.of(HighlightFlag.WEIGHT_MATCHES, HighlightFlag.PHRASES))) {- //assertTrue(snippets[0] + " " + query, snippets[0].contains("<b>Buddhist origins</b>"));+ // highlighter.getFlags("body").containsAll(EnumSet.of(HighlightFlag.WEIGHT_MATCHES,+ // HighlightFlag.PHRASES))) {+ // assertTrue(snippets[0] + " " + query, snippets[0].contains("<b>Buddhist origins</b>"));assertTrue(snippets[0], snippets[0].contains("<b>Buddhist</b> <b>origins</b>"));ir.close();}public void testCuriousGeorge() throws Exception {- String text = "It’s the formula for success for preschoolers—Curious George and fire trucks! " +- "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s " +- "popular primate and painted in the original watercolor and charcoal style. " +- "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";+ String text =+ "It’s the formula for success for preschoolers—Curious George and fire trucks! "+ + "Curious George and the Firefighters is a story based on H. A. and Margret Rey’s "+ + "popular primate and painted in the original watercolor and charcoal style. "+ + "Firefighters are a famously brave lot, but can they withstand a visit from one curious monkey?";RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Field body = new Field("body", text, fieldType);@@ -409,16 +428,18 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {assertFalse(snippets[0].contains("<b>Curious</b>Curious"));int matches = 0;final String snippet = "<b>Curious((</b> <b>)| )?George</b>";- for (Matcher m = Pattern.compile(snippet).matcher(snippets[0]); m.find();) {+ for (Matcher m = Pattern.compile(snippet).matcher(snippets[0]); m.find(); ) {matches++;}- assertEquals(query +" is looking for "+snippet+" at "+snippets[0], 2, matches);+ assertEquals(query + " is looking for " + snippet + " at " + snippets[0], 2, matches);ir.close();}-+public void testCambridgeMA() throws Exception {- BufferedReader r = new BufferedReader(new InputStreamReader(- this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));+ BufferedReader r =+ new BufferedReader(+ new InputStreamReader(+ this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));String text = r.readLine();r.close();RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);@@ -428,21 +449,26 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {iw.addDocument(document);IndexReader ir = iw.getReader();try {- iw.close();- IndexSearcher searcher = newSearcher(ir);- Query query = new IntervalQuery("body",- Intervals.unordered(Intervals.term("porter"),- Intervals.term("square"),- Intervals.term("massachusetts")));- TopDocs topDocs = searcher.search(query, 10);- assertEquals(1, topDocs.totalHits.value);- UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setMaxLength(Integer.MAX_VALUE - 1);- String snippets[] = highlighter.highlight("body", query, topDocs, 2);- assertEquals(1, snippets.length);- assertTrue(snippets[0].contains("<b>Square</b>"));- assertTrue(snippets[0].contains("<b>Porter</b>"));- } finally{ir.close();}+ iw.close();+ IndexSearcher searcher = newSearcher(ir);+ Query query =+ new IntervalQuery(+ "body",+ Intervals.unordered(+ Intervals.term("porter"),+ Intervals.term("square"),+ Intervals.term("massachusetts")));+ TopDocs topDocs = searcher.search(query, 10);+ assertEquals(1, topDocs.totalHits.value);+ UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);+ highlighter.setMaxLength(Integer.MAX_VALUE - 1);+ String snippets[] = highlighter.highlight("body", query, topDocs, 2);+ assertEquals(1, snippets.length);+ assertTrue(snippets[0].contains("<b>Square</b>"));+ assertTrue(snippets[0].contains("<b>Porter</b>"));+ } finally {+ ir.close();+ }}public void testPassageRanking() throws Exception {@@ -452,7 +478,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");+ body.setStringValue(+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");iw.addDocument(doc);IndexReader ir = iw.getReader();@@ -460,7 +487,7 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- Query query = new IntervalQuery("body",Intervals.term("test")) ;+ Query query = new IntervalQuery("body", Intervals.term("test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs, 2);@@ -474,7 +501,9 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {public void testBooleanMustNot() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);- Field body = new Field("body", "This sentence has both terms. This sentence has only terms.", fieldType);+ Field body =+ new Field(+ "body", "This sentence has both terms. This sentence has only terms.", fieldType);Document document = new Document();document.add(body);iw.addDocument(document);@@ -482,8 +511,9 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- Query query = new IntervalQuery("body", Intervals.notContaining(Intervals.term("terms"),- Intervals.term("both")));+ Query query =+ new IntervalQuery(+ "body", Intervals.notContaining(Intervals.term("terms"), Intervals.term("both")));TopDocs topDocs = searcher.search(query, 10);assertEquals(1, topDocs.totalHits.value);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);@@ -502,19 +532,21 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");+ body.setStringValue(+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };highlighter.setMaxLength(10000);Query query = new IntervalQuery("body", Intervals.term("test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);@@ -534,7 +566,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);body.setStringValue("Highlighting the first term. Hope it works.");iw.addDocument(doc);@@ -551,7 +584,10 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {int[] docIDs = new int[2];docIDs[0] = hits[0].doc;docIDs[1] = hits[1].doc;- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {1}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {1})+ .get("body");assertEquals(2, snippets.length);assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);@@ -563,7 +599,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();- final String text = "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";+ final String text =+ "This is a test. Just highlighting from postings. This is also a much sillier test. Feel free to test test test test test test test.";Field body = new Field("body", text, fieldType);doc.add(body);iw.addDocument(doc);@@ -573,21 +610,23 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected List<CharSequence[]> loadFieldValues(String[] fields,- DocIdSetIterator docIter, int cacheCharsThreshold) throws IOException {- assert fields.length == 1;- assert docIter.cost() == 1;- docIter.nextDoc();- return Collections.singletonList(new CharSequence[]{text});- }+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected List<CharSequence[]> loadFieldValues(+ String[] fields, DocIdSetIterator docIter, int cacheCharsThreshold)+ throws IOException {+ assert fields.length == 1;+ assert docIter.cost() == 1;+ docIter.nextDoc();+ return Collections.singletonList(new CharSequence[] {text});+ }- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };Query query = new IntervalQuery("body", Intervals.term("test"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);@@ -600,16 +639,17 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {ir.close();}- /**- * Make sure highlighter returns first N sentences if- * there were no hits.- */+ /** Make sure highlighter returns first N sentences if there were no hits. */public void testEmptyHighlights() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -620,20 +660,21 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);Query query = new IntervalQuery("body", Intervals.term("highlighting"));int[] docIDs = new int[] {0};- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals("test this is. another sentence this test has. ", snippets[0]);ir.close();}- /**- * Not empty but nothing analyzes. Ensures we address null term-vectors.- */+ /** Not empty but nothing analyzes. Ensures we address null term-vectors. */public void testNothingAnalyzes() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- doc.add(new Field("body", " ", fieldType));// just a space! (thus not empty)+ doc.add(new Field("body", " ", fieldType)); // just a space! (thus not empty)doc.add(newTextField("id", "id", Field.Store.YES));iw.addDocument(doc);@@ -650,23 +691,27 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Query query = new IntervalQuery("body", Intervals.term("highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals(" ", snippets[0]);ir.close();}- /**- * Make sure highlighter we can customize how emtpy- * highlight is returned.- */+ /** Make sure highlighter we can customize how emtpy highlight is returned. */public void testCustomEmptyHighlights() throws Exception {indexAnalyzer.setPositionIncrementGap(10);RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -675,25 +720,29 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setMaxNoHighlightPassages(0);// don't want any default summary+ highlighter.setMaxNoHighlightPassages(0); // don't want any default summaryQuery query = new IntervalQuery("body", Intervals.term("highlighting"));- int[] docIDs = new int[]{0};- String snippets[] = highlighter.highlightFields(new String[]{"body"}, query, docIDs, new int[]{2}).get("body");+ int[] docIDs = new int[] {0};+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertNull(snippets[0]);ir.close();}- /**- * Make sure highlighter returns whole text when there- * are no hits and BreakIterator is null.- */+ /** Make sure highlighter returns whole text when there are no hits and BreakIterator is null. */public void testEmptyHighlightsWhole() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -701,30 +750,36 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected BreakIterator getBreakIterator(String field) {- return new WholeBreakIterator();- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected BreakIterator getBreakIterator(String field) {+ return new WholeBreakIterator();+ }+ };Query query = new IntervalQuery("body", Intervals.term("highlighting"));int[] docIDs = new int[] {0};- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);- assertEquals("test this is. another sentence this test has. far away is that planet.", snippets[0]);+ assertEquals(+ "test this is. another sentence this test has. far away is that planet.", snippets[0]);ir.close();}- /**- * Make sure highlighter is OK with entirely missing- * field.- */+ /** Make sure highlighter is OK with entirely missing field. */public void testFieldIsMissing() throws Exception {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);Document doc = new Document();- Field body = new Field("body", "test this is. another sentence this test has. far away is that planet.", fieldType);+ Field body =+ new Field(+ "body",+ "test this is. another sentence this test has. far away is that planet.",+ fieldType);doc.add(body);iw.addDocument(doc);@@ -735,7 +790,10 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);Query query = new IntervalQuery("bogus", Intervals.term("highlighting"));int[] docIDs = new int[] {0};- String snippets[] = highlighter.highlightFields(new String[] {"bogus"}, query, docIDs, new int[] {2}).get("bogus");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"bogus"}, query, docIDs, new int[] {2})+ .get("bogus");assertEquals(1, snippets.length);assertNull(snippets[0]);ir.close();@@ -763,7 +821,10 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Query query = new IntervalQuery("body", Intervals.term("highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertEquals(" ", snippets[0]);ir.close();@@ -791,7 +852,10 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Query query = new IntervalQuery("body", Intervals.term("highlighting"));int[] docIDs = new int[1];docIDs[0] = docID;- String snippets[] = highlighter.highlightFields(new String[] {"body"}, query, docIDs, new int[] {2}).get("body");+ String snippets[] =+ highlighter+ .highlightFields(new String[] {"body"}, query, docIDs, new int[] {2})+ .get("body");assertEquals(1, snippets.length);assertNull(snippets[0]);ir.close();@@ -821,7 +885,8 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);- highlighter.setCacheFieldValCharsThreshold(random().nextInt(10) * 10);// 0 thru 90 intervals of 10+ highlighter.setCacheFieldValCharsThreshold(+ random().nextInt(10) * 10); // 0 thru 90 intervals of 10Query query = new IntervalQuery("body", Intervals.term("answer"));TopDocs hits = searcher.search(query, numDocs);assertEquals(numDocs, hits.totalHits.value);@@ -847,26 +912,29 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from <i>postings</i>. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- return new DefaultPassageFormatter("<b>", "</b>", "... ", true);- }- };-+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected PassageFormatter getFormatter(String field) {+ return new DefaultPassageFormatter("<b>", "</b>", "... ", true);+ }+ };+Query query = new IntervalQuery("body", Intervals.term("highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);assertEquals(1, snippets.length);- assertEquals("Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);+ assertEquals(+ "Just a test <b>highlighting</b> from <i>postings</i>. ", snippets[0]);ir.close();}@@ -878,40 +946,46 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected PassageFormatter getFormatter(String field) {- return new PassageFormatter() {- PassageFormatter defaultFormatter = new DefaultPassageFormatter();-+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {@Override- public String[] format(Passage passages[], String content) {- // Just turns the String snippet into a length 2- // array of String- return new String[]{"blah blah", defaultFormatter.format(passages, content).toString()};+ protected PassageFormatter getFormatter(String field) {+ return new PassageFormatter() {+ PassageFormatter defaultFormatter = new DefaultPassageFormatter();++ @Override+ public String[] format(Passage passages[], String content) {+ // Just turns the String snippet into a length 2+ // array of String+ return new String[] {+ "blah blah", defaultFormatter.format(passages, content).toString()+ };+ }+ };}};- }- };Query query = new IntervalQuery("body", Intervals.term("highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);int[] docIDs = new int[1];docIDs[0] = topDocs.scoreDocs[0].doc;- Map<String,Object[]> snippets = highlighter.highlightFieldsAsObjects(new String[] {"body"}, query, docIDs,- new int[] {1});+ Map<String, Object[]> snippets =+ highlighter.highlightFieldsAsObjects(new String[] {"body"}, query, docIDs, new int[] {1});Object[] bodySnippets = snippets.get("body");assertEquals(1, bodySnippets.length);- assertTrue(Arrays.equals(new String[] {"blah blah", "Just a test <b>highlighting</b> from postings. "},- (String[]) bodySnippets[0]));+ assertTrue(+ Arrays.equals(+ new String[] {"blah blah", "Just a test <b>highlighting</b> from postings. "},+ (String[]) bodySnippets[0]));ir.close();}@@ -945,19 +1019,28 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {IndexReader ir = indexSomeFields();IndexSearcher searcher = newSearcher(ir);UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);- Query query = new IntervalQuery("title", Intervals.maxgaps(random().nextBoolean() ? 1 : 2,- Intervals.ordered(- Intervals.term("this"), Intervals.term("is"), Intervals.term("the"), Intervals.term("field"))));+ Query query =+ new IntervalQuery(+ "title",+ Intervals.maxgaps(+ random().nextBoolean() ? 1 : 2,+ Intervals.ordered(+ Intervals.term("this"),+ Intervals.term("is"),+ Intervals.term("the"),+ Intervals.term("field"))));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String[] snippets = highlighter.highlight("title", query, topDocs, 10);assertEquals(1, snippets.length);if (highlighter.getFlags("title").contains(HighlightFlag.WEIGHT_MATCHES)) {- assertEquals("" + highlighter.getFlags("title"),- "<b>This is the title field</b>.", snippets[0]);+ assertEquals(+ "" + highlighter.getFlags("title"), "<b>This is the title field</b>.", snippets[0]);} else {- assertEquals("" + highlighter.getFlags("title"),- "<b>This</b> <b>is</b> <b>the</b> title <b>field</b>.", snippets[0]);+ assertEquals(+ "" + highlighter.getFlags("title"),+ "<b>This</b> <b>is</b> <b>the</b> title <b>field</b>.",+ snippets[0]);}ir.close();}@@ -973,20 +1056,24 @@ public class TestUnifiedHighlighterTermIntervals extends LuceneTestCase {Document doc = new Document();doc.add(body);- body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");+ body.setStringValue(+ "This is a test. Just a test highlighting from postings. Feel free to ignore.");iw.addDocument(doc);IndexReader ir = iw.getReader();iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, new Analyzer() {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- throw new AssertionError("shouldn't be called");- }- });- Query query = new IntervalQuery("body", Intervals.term("highlighting"));+ UnifiedHighlighter highlighter =+ randomUnifiedHighlighter(+ searcher,+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ throw new AssertionError("shouldn't be called");+ }+ });+ Query query = new IntervalQuery("body", Intervals.term("highlighting"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);assertEquals(1, topDocs.totalHits.value);String snippets[] = highlighter.highlight("body", query, topDocs);diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.javaindex f929f20f473..002855e9a1d 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighterTermVec.java@@ -24,7 +24,6 @@ import java.util.Collections;import java.util.List;import java.util.Map;import java.util.Set;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.analysis.MockTokenizer;@@ -55,9 +54,10 @@ import org.junit.Test;/*** Tests highlighting for matters *expressly* relating to term vectors.- * <p>- * This test DOES NOT represent all testing for highlighting when term vectors are used. Other tests pick the offset- * source at random (to include term vectors) and in-effect test term vectors generally.+ *+ * <p>This test DOES NOT represent all testing for highlighting when term vectors are used. Other+ * tests pick the offset source at random (to include term vectors) and in-effect test term vectors+ * generally.*/public class TestUnifiedHighlighterTermVec extends LuceneTestCase {@@ -66,7 +66,9 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {@Beforepublic void doBefore() throws IOException {- indexAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);//whitespace, punctuation, lowercase+ indexAnalyzer =+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true); // whitespace, punctuation, lowercasedir = newDirectory();}@@ -78,7 +80,8 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {public void testFetchTermVecsOncePerDoc() throws IOException {RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);- // Declare some number of fields with random field type; but at least one will have term vectors.+ // Declare some number of fields with random field type; but at least one will have term+ // vectors.final int numTvFields = 1 + random().nextInt(3);List<String> fields = new ArrayList<>(numTvFields);List<FieldType> fieldTypes = new ArrayList<>(numTvFields);@@ -86,7 +89,7 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {fields.add("body" + i);fieldTypes.add(UHTestHelper.randomFieldType(random()));}- //ensure at least one has TVs by setting one randomly to it:+ // ensure at least one has TVs by setting one randomly to it:fieldTypes.set(random().nextInt(fieldTypes.size()), UHTestHelper.tvType);final int numDocs = 1 + random().nextInt(3);@@ -125,36 +128,39 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {}private static class AssertOnceTermVecDirectoryReader extends FilterDirectoryReader {- static final SubReaderWrapper SUB_READER_WRAPPER = new SubReaderWrapper() {- @Override- public LeafReader wrap(LeafReader reader) {- return new FilterLeafReader(reader) {- BitSet seenDocIDs = new BitSet();-- @Override- public Fields getTermVectors(int docID) throws IOException {- // if we're invoked by ParallelLeafReader then we can't do our assertion. TODO see LUCENE-6868- if (callStackContains(ParallelLeafReader.class) == false- && callStackContains(CheckIndex.class) == false) {- assertFalse("Should not request TVs for doc more than once.", seenDocIDs.get(docID));- seenDocIDs.set(docID);- }-- return super.getTermVectors(docID);- }-+ static final SubReaderWrapper SUB_READER_WRAPPER =+ new SubReaderWrapper() {@Override- public CacheHelper getCoreCacheHelper() {- return null;- }-- @Override- public CacheHelper getReaderCacheHelper() {- return null;+ public LeafReader wrap(LeafReader reader) {+ return new FilterLeafReader(reader) {+ BitSet seenDocIDs = new BitSet();++ @Override+ public Fields getTermVectors(int docID) throws IOException {+ // if we're invoked by ParallelLeafReader then we can't do our assertion. TODO see+ // LUCENE-6868+ if (callStackContains(ParallelLeafReader.class) == false+ && callStackContains(CheckIndex.class) == false) {+ assertFalse(+ "Should not request TVs for doc more than once.", seenDocIDs.get(docID));+ seenDocIDs.set(docID);+ }++ return super.getTermVectors(docID);+ }++ @Override+ public CacheHelper getCoreCacheHelper() {+ return null;+ }++ @Override+ public CacheHelper getReaderCacheHelper() {+ return null;+ }+ };}};- }- };AssertOnceTermVecDirectoryReader(DirectoryReader in) throws IOException {super(in, SUB_READER_WRAPPER);@@ -186,19 +192,19 @@ public class TestUnifiedHighlighterTermVec extends LuceneTestCase {iw.close();IndexSearcher searcher = newSearcher(ir);- UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer) {- @Override- protected Set<HighlightFlag> getFlags(String field) {- return Collections.emptySet();//no WEIGHT_MATCHES- }- };+ UnifiedHighlighter highlighter =+ new UnifiedHighlighter(searcher, indexAnalyzer) {+ @Override+ protected Set<HighlightFlag> getFlags(String field) {+ return Collections.emptySet(); // no WEIGHT_MATCHES+ }+ };TermQuery query = new TermQuery(new Term("body", "vectors"));TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);try {- highlighter.highlight("body", query, topDocs, 1);//should throw+ highlighter.highlight("body", query, topDocs, 1); // should throw} finally {ir.close();}}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.javaindex 8e5b2ff13af..16dcd49fdea 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestWholeBreakIterator.java@@ -20,12 +20,13 @@ import java.text.BreakIterator;import java.text.CharacterIterator;import java.text.StringCharacterIterator;import java.util.Locale;-import org.apache.lucene.util.LuceneTestCase;public class TestWholeBreakIterator extends LuceneTestCase {-- /** For single sentences, we know WholeBreakIterator should break the same as a sentence iterator */++ /**+ * For single sentences, we know WholeBreakIterator should break the same as a sentence iterator+ */public void testSingleSentences() throws Exception {BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);BreakIterator actual = new WholeBreakIterator();@@ -34,7 +35,7 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertSameBreaks("abc", expected, actual);assertSameBreaks("", expected, actual);}-+public void testSliceEnd() throws Exception {BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);BreakIterator actual = new WholeBreakIterator();@@ -43,7 +44,7 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertSameBreaks("abc000", 0, 1, expected, actual);assertSameBreaks("000", 0, 0, expected, actual);}-+public void testSliceStart() throws Exception {BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);BreakIterator actual = new WholeBreakIterator();@@ -52,7 +53,7 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertSameBreaks("000abc", 3, 3, expected, actual);assertSameBreaks("000", 3, 0, expected, actual);}-+public void testSliceMiddle() throws Exception {BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);BreakIterator actual = new WholeBreakIterator();@@ -61,7 +62,7 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertSameBreaks("000abc000", 3, 3, expected, actual);assertSameBreaks("000000", 3, 0, expected, actual);}-+/** the current position must be ignored, initial position is always first() */public void testFirstPosition() throws Exception {BreakIterator expected = BreakIterator.getSentenceInstance(Locale.ROOT);@@ -70,25 +71,32 @@ public class TestWholeBreakIterator extends LuceneTestCase {}public static void assertSameBreaks(String text, BreakIterator expected, BreakIterator actual) {- assertSameBreaks(new StringCharacterIterator(text),- new StringCharacterIterator(text),- expected,- actual);+ assertSameBreaks(+ new StringCharacterIterator(text), new StringCharacterIterator(text), expected, actual);}-- public static void assertSameBreaks(String text, int offset, int length, BreakIterator expected, BreakIterator actual) {++ public static void assertSameBreaks(+ String text, int offset, int length, BreakIterator expected, BreakIterator actual) {assertSameBreaks(text, offset, length, offset, expected, actual);}-- public static void assertSameBreaks(String text, int offset, int length, int current, BreakIterator expected, BreakIterator actual) {- assertSameBreaks(new StringCharacterIterator(text, offset, offset+length, current),- new StringCharacterIterator(text, offset, offset+length, current),- expected,- actual);++ public static void assertSameBreaks(+ String text,+ int offset,+ int length,+ int current,+ BreakIterator expected,+ BreakIterator actual) {+ assertSameBreaks(+ new StringCharacterIterator(text, offset, offset + length, current),+ new StringCharacterIterator(text, offset, offset + length, current),+ expected,+ actual);}/** Asserts that two breakiterators break the text the same way */- public static void assertSameBreaks(CharacterIterator one, CharacterIterator two, BreakIterator expected, BreakIterator actual) {+ public static void assertSameBreaks(+ CharacterIterator one, CharacterIterator two, BreakIterator expected, BreakIterator actual) {expected.setText(one);actual.setText(two);@@ -100,21 +108,21 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertEquals(v = expected.next(), actual.next());assertEquals(expected.current(), actual.current());}-+// first()assertEquals(expected.first(), actual.first());assertEquals(expected.current(), actual.current());// last()assertEquals(expected.last(), actual.last());assertEquals(expected.current(), actual.current());-+// previous()v = expected.current();while (v != BreakIterator.DONE) {assertEquals(v = expected.previous(), actual.previous());assertEquals(expected.current(), actual.current());}-+// following()for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {expected.first();@@ -122,7 +130,7 @@ public class TestWholeBreakIterator extends LuceneTestCase {assertEquals(expected.following(i), actual.following(i));assertEquals(expected.current(), actual.current());}-+// preceding()for (int i = one.getBeginIndex(); i <= one.getEndIndex(); i++) {expected.last();diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/UHTestHelper.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/UHTestHelper.javaindex 67400bee5f7..e4a9558d04a 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/UHTestHelper.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/UHTestHelper.java@@ -18,14 +18,11 @@ package org.apache.lucene.search.uhighlight;import java.util.Arrays;import java.util.Random;-import org.apache.lucene.document.FieldType;import org.apache.lucene.document.TextField;import org.apache.lucene.index.IndexOptions;-/**- * Helper for {@link UnifiedHighlighter} tests.- */+/** Helper for {@link UnifiedHighlighter} tests. */class UHTestHelper {static final FieldType postingsType = new FieldType(TextField.TYPE_STORED);@@ -46,24 +43,21 @@ class UHTestHelper {postingsWithTvType.setStoreTermVectors(true);postingsWithTvType.freeze();-- //re-analysis type needs no further changes.+ // re-analysis type needs no further changes.}public static FieldType randomFieldType(Random random, FieldType... typePossibilities) {if (typePossibilities == null || typePossibilities.length == 0) {- typePossibilities = new FieldType[]{postingsType, tvType, postingsWithTvType, reanalysisType};+ typePossibilities =+ new FieldType[] {postingsType, tvType, postingsWithTvType, reanalysisType};}return typePossibilities[random.nextInt(typePossibilities.length)];}- /**- * for {@link com.carrotsearch.randomizedtesting.annotations.ParametersFactory}- */+ /** for {@link com.carrotsearch.randomizedtesting.annotations.ParametersFactory} */// https://github.com/carrotsearch/randomizedtesting/blob/master/examples/maven/src/main/java/com/carrotsearch/examples/randomizedrunner/Test007ParameterizedTests.javastatic Iterable<Object[]> parametersFactoryList() {- return Arrays.asList(new Object[][]{- {postingsType}, {tvType}, {postingsWithTvType}, {reanalysisType}- });+ return Arrays.asList(+ new Object[][] {{postingsType}, {tvType}, {postingsWithTvType}, {reanalysisType}});}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.javaindex 33bc7e1d2a0..4cff4ae13b1 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/visibility/TestUnifiedHighlighterExtensibility.java@@ -24,7 +24,6 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.function.Predicate;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.index.LeafReader;@@ -49,188 +48,232 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.LuceneTestCase;import org.junit.Test;-/**- * Helps us be aware of visibility/extensibility concerns.- */+/** Helps us be aware of visibility/extensibility concerns. */public class TestUnifiedHighlighterExtensibility extends LuceneTestCase {/**- * This test is for maintaining the extensibility of the FieldOffsetStrategy- * for customizations out of package.+ * This test is for maintaining the extensibility of the FieldOffsetStrategy for customizations+ * out of package.*/@Testpublic void testFieldOffsetStrategyExtensibility() {- final UnifiedHighlighter.OffsetSource offsetSource = UnifiedHighlighter.OffsetSource.NONE_NEEDED;- FieldOffsetStrategy strategy = new FieldOffsetStrategy(new UHComponents("field",- (s) -> false,- new MatchAllDocsQuery(), new BytesRef[0],- PhraseHelper.NONE,- new LabelledCharArrayMatcher[0], false, Collections.emptySet())) {- @Override- public UnifiedHighlighter.OffsetSource getOffsetSource() {- return offsetSource;- }-- @Override- public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content) throws IOException {- return OffsetsEnum.EMPTY;- }-- @Override- protected OffsetsEnum createOffsetsEnumFromReader(LeafReader leafReader, int doc) throws IOException {- return super.createOffsetsEnumFromReader(leafReader, doc);- }-- };+ final UnifiedHighlighter.OffsetSource offsetSource =+ UnifiedHighlighter.OffsetSource.NONE_NEEDED;+ FieldOffsetStrategy strategy =+ new FieldOffsetStrategy(+ new UHComponents(+ "field",+ (s) -> false,+ new MatchAllDocsQuery(),+ new BytesRef[0],+ PhraseHelper.NONE,+ new LabelledCharArrayMatcher[0],+ false,+ Collections.emptySet())) {+ @Override+ public UnifiedHighlighter.OffsetSource getOffsetSource() {+ return offsetSource;+ }++ @Override+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)+ throws IOException {+ return OffsetsEnum.EMPTY;+ }++ @Override+ protected OffsetsEnum createOffsetsEnumFromReader(LeafReader leafReader, int doc)+ throws IOException {+ return super.createOffsetsEnumFromReader(leafReader, doc);+ }+ };assertEquals(offsetSource, strategy.getOffsetSource());}/**- * This test is for maintaining the extensibility of the UnifiedHighlighter- * for customizations out of package.+ * This test is for maintaining the extensibility of the UnifiedHighlighter for customizations out+ * of package.*/@Testpublic void testUnifiedHighlighterExtensibility() {final int maxLength = 1000;- UnifiedHighlighter uh = new UnifiedHighlighter(null, new MockAnalyzer(random())){-- @Override- protected Map<String, Object[]> highlightFieldsAsObjects(String[] fieldsIn, Query query, int[] docIdsIn, int[] maxPassagesIn) throws IOException {- return super.highlightFieldsAsObjects(fieldsIn, query, docIdsIn, maxPassagesIn);- }-- @Override- protected OffsetSource getOffsetSource(String field) {- return super.getOffsetSource(field);- }-- @Override- protected BreakIterator getBreakIterator(String field) {- return super.getBreakIterator(field);- }-- @Override- protected PassageScorer getScorer(String field) {- return super.getScorer(field);- }-- @Override- protected PassageFormatter getFormatter(String field) {- return super.getFormatter(field);- }-- @Override- public Analyzer getIndexAnalyzer() {- return super.getIndexAnalyzer();- }-- @Override- public IndexSearcher getIndexSearcher() {- return super.getIndexSearcher();- }-- @Override- protected int getMaxNoHighlightPassages(String field) {- return super.getMaxNoHighlightPassages(field);- }-- @Override- protected Boolean requiresRewrite(SpanQuery spanQuery) {- return super.requiresRewrite(spanQuery);- }-- @Override- protected LimitedStoredFieldVisitor newLimitedStoredFieldsVisitor(String[] fields) {- return super.newLimitedStoredFieldsVisitor(fields);- }-- @Override- protected List<CharSequence[]> loadFieldValues(String[] fields, DocIdSetIterator docIter, int cacheCharsThreshold) throws IOException {- return super.loadFieldValues(fields, docIter, cacheCharsThreshold);- }-- @Override- protected FieldHighlighter getFieldHighlighter(String field, Query query, Set<Term> allTerms, int maxPassages) {- // THIS IS A COPY of the superclass impl; but use CustomFieldHighlighter- UHComponents components = getHighlightComponents(field, query, allTerms);- OffsetSource offsetSource = getOptimizedOffsetSource(components);-- // test all is accessible- components.getField();- components.getFieldMatcher();- components.getQuery();- components.getTerms();- components.getPhraseHelper();- components.getAutomata();- components.hasUnrecognizedQueryPart();- components.getHighlightFlags();-- return new CustomFieldHighlighter(field,- getOffsetStrategy(offsetSource, components),- new SplittingBreakIterator(getBreakIterator(field), UnifiedHighlighter.MULTIVAL_SEP_CHAR),- getScorer(field),- maxPassages,- getMaxNoHighlightPassages(field),- getFormatter(field));- }-- @Override- protected UHComponents getHighlightComponents(String field, Query query, Set<Term> allTerms) {- Predicate<String> fieldMatcher = getFieldMatcher(field);- BytesRef[] terms = filterExtractedTerms(fieldMatcher, allTerms);- Set<HighlightFlag> highlightFlags = getFlags(field);- PhraseHelper phraseHelper = getPhraseHelper(field, query, highlightFlags);- LabelledCharArrayMatcher[] automata = getAutomata(field, query, highlightFlags);- boolean queryHasUnrecognizedPart = false;- return new UHComponents(field, fieldMatcher, query, terms, phraseHelper, automata, queryHasUnrecognizedPart, highlightFlags);- }-- @Override- protected FieldOffsetStrategy getOffsetStrategy(OffsetSource offsetSource, UHComponents components) {- return super.getOffsetStrategy(offsetSource, components);- }-- @Override- public int getMaxLength() {- return maxLength;- }- };+ UnifiedHighlighter uh =+ new UnifiedHighlighter(null, new MockAnalyzer(random())) {++ @Override+ protected Map<String, Object[]> highlightFieldsAsObjects(+ String[] fieldsIn, Query query, int[] docIdsIn, int[] maxPassagesIn)+ throws IOException {+ return super.highlightFieldsAsObjects(fieldsIn, query, docIdsIn, maxPassagesIn);+ }++ @Override+ protected OffsetSource getOffsetSource(String field) {+ return super.getOffsetSource(field);+ }++ @Override+ protected BreakIterator getBreakIterator(String field) {+ return super.getBreakIterator(field);+ }++ @Override+ protected PassageScorer getScorer(String field) {+ return super.getScorer(field);+ }++ @Override+ protected PassageFormatter getFormatter(String field) {+ return super.getFormatter(field);+ }++ @Override+ public Analyzer getIndexAnalyzer() {+ return super.getIndexAnalyzer();+ }++ @Override+ public IndexSearcher getIndexSearcher() {+ return super.getIndexSearcher();+ }++ @Override+ protected int getMaxNoHighlightPassages(String field) {+ return super.getMaxNoHighlightPassages(field);+ }++ @Override+ protected Boolean requiresRewrite(SpanQuery spanQuery) {+ return super.requiresRewrite(spanQuery);+ }++ @Override+ protected LimitedStoredFieldVisitor newLimitedStoredFieldsVisitor(String[] fields) {+ return super.newLimitedStoredFieldsVisitor(fields);+ }++ @Override+ protected List<CharSequence[]> loadFieldValues(+ String[] fields, DocIdSetIterator docIter, int cacheCharsThreshold)+ throws IOException {+ return super.loadFieldValues(fields, docIter, cacheCharsThreshold);+ }++ @Override+ protected FieldHighlighter getFieldHighlighter(+ String field, Query query, Set<Term> allTerms, int maxPassages) {+ // THIS IS A COPY of the superclass impl; but use CustomFieldHighlighter+ UHComponents components = getHighlightComponents(field, query, allTerms);+ OffsetSource offsetSource = getOptimizedOffsetSource(components);++ // test all is accessible+ components.getField();+ components.getFieldMatcher();+ components.getQuery();+ components.getTerms();+ components.getPhraseHelper();+ components.getAutomata();+ components.hasUnrecognizedQueryPart();+ components.getHighlightFlags();++ return new CustomFieldHighlighter(+ field,+ getOffsetStrategy(offsetSource, components),+ new SplittingBreakIterator(+ getBreakIterator(field), UnifiedHighlighter.MULTIVAL_SEP_CHAR),+ getScorer(field),+ maxPassages,+ getMaxNoHighlightPassages(field),+ getFormatter(field));+ }++ @Override+ protected UHComponents getHighlightComponents(+ String field, Query query, Set<Term> allTerms) {+ Predicate<String> fieldMatcher = getFieldMatcher(field);+ BytesRef[] terms = filterExtractedTerms(fieldMatcher, allTerms);+ Set<HighlightFlag> highlightFlags = getFlags(field);+ PhraseHelper phraseHelper = getPhraseHelper(field, query, highlightFlags);+ LabelledCharArrayMatcher[] automata = getAutomata(field, query, highlightFlags);+ boolean queryHasUnrecognizedPart = false;+ return new UHComponents(+ field,+ fieldMatcher,+ query,+ terms,+ phraseHelper,+ automata,+ queryHasUnrecognizedPart,+ highlightFlags);+ }++ @Override+ protected FieldOffsetStrategy getOffsetStrategy(+ OffsetSource offsetSource, UHComponents components) {+ return super.getOffsetStrategy(offsetSource, components);+ }++ @Override+ public int getMaxLength() {+ return maxLength;+ }+ };assertEquals(uh.getMaxLength(), maxLength);}@Testpublic void testPassageFormatterExtensibility() {final Object formattedResponse = new Object();- PassageFormatter formatter = new PassageFormatter() {- @Override- public Object format(Passage[] passages, String content) {- return formattedResponse;- }- };+ PassageFormatter formatter =+ new PassageFormatter() {+ @Override+ public Object format(Passage[] passages, String content) {+ return formattedResponse;+ }+ };assertEquals(formattedResponse, formatter.format(new Passage[0], ""));}@Testpublic void testFieldHiglighterExtensibility() {final String fieldName = "fieldName";- FieldHighlighter fieldHighlighter = new FieldHighlighter(fieldName, null, null, null, 1, 1, null) {- @Override- protected Passage[] highlightOffsetsEnums(OffsetsEnum offsetsEnums) throws IOException {- return super.highlightOffsetsEnums(offsetsEnums);- }- };+ FieldHighlighter fieldHighlighter =+ new FieldHighlighter(fieldName, null, null, null, 1, 1, null) {+ @Override+ protected Passage[] highlightOffsetsEnums(OffsetsEnum offsetsEnums) throws IOException {+ return super.highlightOffsetsEnums(offsetsEnums);+ }+ };assertEquals(fieldHighlighter.getField(), fieldName);}- /** Tests maintaining extensibility/visibility of {@link org.apache.lucene.search.uhighlight.FieldHighlighter} out of package. */+ /**+ * Tests maintaining extensibility/visibility of {@link+ * org.apache.lucene.search.uhighlight.FieldHighlighter} out of package.+ */private static class CustomFieldHighlighter extends FieldHighlighter {- CustomFieldHighlighter(String field, FieldOffsetStrategy fieldOffsetStrategy, BreakIterator breakIterator, PassageScorer passageScorer, int maxPassages, int maxNoHighlightPassages, PassageFormatter passageFormatter) {- super(field, fieldOffsetStrategy, breakIterator, passageScorer, maxPassages, maxNoHighlightPassages, passageFormatter);+ CustomFieldHighlighter(+ String field,+ FieldOffsetStrategy fieldOffsetStrategy,+ BreakIterator breakIterator,+ PassageScorer passageScorer,+ int maxPassages,+ int maxNoHighlightPassages,+ PassageFormatter passageFormatter) {+ super(+ field,+ fieldOffsetStrategy,+ breakIterator,+ passageScorer,+ maxPassages,+ maxNoHighlightPassages,+ passageFormatter);}@Override- public Object highlightFieldForDoc(LeafReader reader, int docId, String content) throws IOException {+ public Object highlightFieldForDoc(LeafReader reader, int docId, String content)+ throws IOException {return super.highlightFieldForDoc(reader, docId, content);}@@ -254,10 +297,9 @@ public class TestUnifiedHighlighterExtensibility extends LuceneTestCase {p.addMatch(1, 2, new BytesRef(), 1);p.reset();p.setScore(1);- //... getters are all exposed; custom PassageFormatter impls uses them+ // ... getters are all exposed; custom PassageFormatter impls uses themreturn super.highlightOffsetsEnums(offsetsEnums);}}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.javaindex 24242196f63..cfb66c1f2d1 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java@@ -21,7 +21,6 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;import java.util.List;-import org.apache.lucene.analysis.*;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;@@ -33,8 +32,8 @@ import org.apache.lucene.document.TextField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig.OpenMode;import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.IndexWriterConfig.OpenMode;import org.apache.lucene.index.Term;import org.apache.lucene.search.BoostQuery;import org.apache.lucene.search.DisjunctionMaxQuery;@@ -54,32 +53,25 @@ public abstract class AbstractTestCase extends LuceneTestCase {protected Analyzer analyzerW;protected Analyzer analyzerB;protected Analyzer analyzerK;- protected IndexReader reader;+ protected IndexReader reader;protected static final String[] shortMVValues = {- "",- "",- "a b c",- "", // empty data in multi valued field+ "", "", "a b c", "", // empty data in multi valued field"d e"};-+protected static final String[] longMVValues = {"Followings are the examples of customizable parameters and actual examples of customization:","The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically"};-+// test data for LUCENE-1448 bugprotected static final String[] biMVValues = {"\nLucene/Solr does not require such additional hardware.","\nWhen you talk about processing speed, the"};-- protected static final String[] strMVValues = {- "abc",- "defg",- "hijkl"- };++ protected static final String[] strMVValues = {"abc", "defg", "hijkl"};@Overridepublic void setUp() throws Exception {@@ -89,10 +81,10 @@ public abstract class AbstractTestCase extends LuceneTestCase {analyzerK = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);dir = newDirectory();}-+@Overridepublic void tearDown() throws Exception {- if( reader != null ){+ if (reader != null) {reader.close();reader = null;}@@ -100,77 +92,79 @@ public abstract class AbstractTestCase extends LuceneTestCase {super.tearDown();}- protected Query tq( String text ){- return tq( 1F, text );+ protected Query tq(String text) {+ return tq(1F, text);}- protected Query tq( float boost, String text ){- return tq( boost, F, text );+ protected Query tq(float boost, String text) {+ return tq(boost, F, text);}-- protected Query tq( String field, String text ){- return tq( 1F, field, text );++ protected Query tq(String field, String text) {+ return tq(1F, field, text);}-- protected Query tq( float boost, String field, String text ){- Query query = new TermQuery( new Term( field, text ) );++ protected Query tq(float boost, String field, String text) {+ Query query = new TermQuery(new Term(field, text));if (boost != 1f) {- query = new BoostQuery( query, boost );+ query = new BoostQuery(query, boost);}return query;}-- protected Query pqF( String... texts ){- return pqF( 1F, texts );++ protected Query pqF(String... texts) {+ return pqF(1F, texts);}-- protected Query pqF( float boost, String... texts ){- return pqF( boost, 0, texts );++ protected Query pqF(float boost, String... texts) {+ return pqF(boost, 0, texts);}-- protected Query pqF( float boost, int slop, String... texts ){- return pq( boost, slop, F, texts );++ protected Query pqF(float boost, int slop, String... texts) {+ return pq(boost, slop, F, texts);}-- protected Query pq( String field, String... texts ){- return pq( 1F, 0, field, texts );++ protected Query pq(String field, String... texts) {+ return pq(1F, 0, field, texts);}-- protected Query pq( float boost, String field, String... texts ){- return pq( boost, 0, field, texts );++ protected Query pq(float boost, String field, String... texts) {+ return pq(boost, 0, field, texts);}-- protected Query pq( float boost, int slop, String field, String... texts ){++ protected Query pq(float boost, int slop, String field, String... texts) {Query query = new PhraseQuery(slop, field, texts);if (boost != 1f) {query = new BoostQuery(query, boost);}return query;}-- protected Query dmq( Query... queries ){- return dmq( 0.0F, queries );++ protected Query dmq(Query... queries) {+ return dmq(0.0F, queries);}-- protected Query dmq( float tieBreakerMultiplier, Query... queries ){++ protected Query dmq(float tieBreakerMultiplier, Query... queries) {return new DisjunctionMaxQuery(Arrays.asList(queries), tieBreakerMultiplier);}-- protected void assertCollectionQueries( Collection<Query> actual, Query... expected ){- assertEquals( expected.length, actual.size() );- for( Query query : expected ){- assertTrue( actual.contains( query ) );++ protected void assertCollectionQueries(Collection<Query> actual, Query... expected) {+ assertEquals(expected.length, actual.size());+ for (Query query : expected) {+ assertTrue(actual.contains(query));}}- protected List<BytesRef> analyze(String text, String field, Analyzer analyzer) throws IOException {+ protected List<BytesRef> analyze(String text, String field, Analyzer analyzer)+ throws IOException {List<BytesRef> bytesRefs = new ArrayList<>();try (TokenStream tokenStream = analyzer.tokenStream(field, text)) {- TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);-+ TermToBytesRefAttribute termAttribute =+ tokenStream.getAttribute(TermToBytesRefAttribute.class);+tokenStream.reset();-+while (tokenStream.incrementToken()) {bytesRefs.add(BytesRef.deepCopyOf(termAttribute.getBytesRef()));}@@ -191,7 +185,7 @@ public abstract class AbstractTestCase extends LuceneTestCase {return new TokenStreamComponents(new BasicNGramTokenizer());}}-+static final class BasicNGramTokenizer extends Tokenizer {public static final int DEFAULT_N_SIZE = 2;@@ -209,20 +203,20 @@ public abstract class AbstractTestCase extends LuceneTestCase {private char[] charBuffer;private int charBufferIndex;private int charBufferLen;-- public BasicNGramTokenizer( ){- this( DEFAULT_N_SIZE );++ public BasicNGramTokenizer() {+ this(DEFAULT_N_SIZE);}-- public BasicNGramTokenizer( int n ){- this( n, DEFAULT_DELIMITERS );++ public BasicNGramTokenizer(int n) {+ this(n, DEFAULT_DELIMITERS);}-- public BasicNGramTokenizer( String delimiters ){- this( DEFAULT_N_SIZE, delimiters );++ public BasicNGramTokenizer(String delimiters) {+ this(DEFAULT_N_SIZE, delimiters);}-- public BasicNGramTokenizer(int n, String delimiters ){++ public BasicNGramTokenizer(int n, String delimiters) {super();this.n = n;this.delimiters = delimiters;@@ -238,10 +232,10 @@ public abstract class AbstractTestCase extends LuceneTestCase {CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);+@Overridepublic boolean incrementToken() throws IOException {- if( !getNextPartialSnippet() )- return false;+ if (!getNextPartialSnippet()) return false;clearAttributes();termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm);offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));@@ -251,15 +245,15 @@ public abstract class AbstractTestCase extends LuceneTestCase {private int getFinalOffset() {return nextStartOffset;}-+@Overridepublic final void end() throws IOException {super.end();- offsetAtt.setOffset(getFinalOffset(),getFinalOffset());+ offsetAtt.setOffset(getFinalOffset(), getFinalOffset());}-+protected boolean getNextPartialSnippet() throws IOException {- if( snippet != null && snippet.length() >= startTerm + 1 + n ){+ if (snippet != null && snippet.length() >= startTerm + 1 + n) {startTerm++;startOffset++;lenTerm = n;@@ -267,33 +261,28 @@ public abstract class AbstractTestCase extends LuceneTestCase {}return getNextSnippet();}-+protected boolean getNextSnippet() throws IOException {startTerm = 0;startOffset = nextStartOffset;- snippetBuffer.delete( 0, snippetBuffer.length() );- while( true ){- if( ch != -1 )- ch = readCharFromBuffer();- if( ch == -1 ) break;- else if( !isDelimiter( ch ) )- snippetBuffer.append( (char)ch );- else if( snippetBuffer.length() > 0 )- break;- else- startOffset++;+ snippetBuffer.delete(0, snippetBuffer.length());+ while (true) {+ if (ch != -1) ch = readCharFromBuffer();+ if (ch == -1) break;+ else if (!isDelimiter(ch)) snippetBuffer.append((char) ch);+ else if (snippetBuffer.length() > 0) break;+ else startOffset++;}- if( snippetBuffer.length() == 0 )- return false;+ if (snippetBuffer.length() == 0) return false;snippet = snippetBuffer.toString();lenTerm = snippet.length() >= n ? n : snippet.length();return true;}-+protected int readCharFromBuffer() throws IOException {- if( charBufferIndex >= charBufferLen ){- charBufferLen = input.read( charBuffer );- if( charBufferLen == -1 ){+ if (charBufferIndex >= charBufferLen) {+ charBufferLen = input.read(charBuffer);+ if (charBufferLen == -1) {return -1;}charBufferIndex = 0;@@ -302,77 +291,80 @@ public abstract class AbstractTestCase extends LuceneTestCase {nextStartOffset++;return c;}-- protected boolean isDelimiter( int c ){- return delimiters.indexOf( c ) >= 0;++ protected boolean isDelimiter(int c) {+ return delimiters.indexOf(c) >= 0;}-+@Overridepublic void reset() throws IOException {super.reset();startTerm = 0;nextStartOffset = 0;snippet = null;- snippetBuffer.setLength( 0 );+ snippetBuffer.setLength(0);charBufferIndex = BUFFER_SIZE;charBufferLen = 0;ch = 0;}}- protected void make1d1fIndex( String value ) throws Exception {- make1dmfIndex( value );+ protected void make1d1fIndex(String value) throws Exception {+ make1dmfIndex(value);}-- protected void make1d1fIndexB( String value ) throws Exception {- make1dmfIndexB( value );++ protected void make1d1fIndexB(String value) throws Exception {+ make1dmfIndexB(value);}-- protected void make1dmfIndex( String... values ) throws Exception {- make1dmfIndex( analyzerW, values );++ protected void make1dmfIndex(String... values) throws Exception {+ make1dmfIndex(analyzerW, values);}-- protected void make1dmfIndexB( String... values ) throws Exception {- make1dmfIndex( analyzerB, values );++ protected void make1dmfIndexB(String... values) throws Exception {+ make1dmfIndex(analyzerB, values);}-+// make 1 doc with multi valued field- protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE));+ protected void make1dmfIndex(Analyzer analyzer, String... values) throws Exception {+ IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(analyzer).setOpenMode(OpenMode.CREATE));Document doc = new Document();FieldType customType = new FieldType(TextField.TYPE_STORED);customType.setStoreTermVectors(true);customType.setStoreTermVectorOffsets(true);customType.setStoreTermVectorPositions(true);- for( String value: values ) {- doc.add( new Field( F, value, customType) );+ for (String value : values) {+ doc.add(new Field(F, value, customType));}- writer.addDocument( doc );+ writer.addDocument(doc);writer.close();if (reader != null) reader.close();reader = DirectoryReader.open(dir);}-+// make 1 doc with multi valued & not analyzed field- protected void make1dmfIndexNA( String... values ) throws Exception {- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzerK).setOpenMode(OpenMode.CREATE));+ protected void make1dmfIndexNA(String... values) throws Exception {+ IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(analyzerK).setOpenMode(OpenMode.CREATE));Document doc = new Document();FieldType customType = new FieldType(TextField.TYPE_STORED);customType.setStoreTermVectors(true);customType.setStoreTermVectorOffsets(true);customType.setStoreTermVectorPositions(true);- for( String value: values ) {- doc.add( new Field( F, value, customType));- //doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );+ for (String value : values) {+ doc.add(new Field(F, value, customType));+ // doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED,+ // TermVector.WITH_POSITIONS_OFFSETS ) );}- writer.addDocument( doc );+ writer.addDocument(doc);writer.close();if (reader != null) reader.close();reader = DirectoryReader.open(dir);}-+protected void makeIndexShortMV() throws Exception {-+// 0// ""// 1@@ -389,27 +381,27 @@ public abstract class AbstractTestCase extends LuceneTestCase {// 9012// "d e"// 3 4- make1dmfIndex( shortMVValues );+ make1dmfIndex(shortMVValues);}-+protected void makeIndexLongMV() throws Exception {// 11111111112222222222333333333344444444445555555555666666666677777777778888888888999// 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012// Followings are the examples of customizable parameters and actual examples of customization:// 0 1 2 3 4 5 6 7 8 9 10 11-- // 1 2++ // _______1___________________________________________________________________________________________________2// 999999900000000001111111111222222222233333333334444444444555555555566666666667777777777888888888899999999990000000000111111111122// 345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901- // The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically- // 12 13 (14) (15) 16 17 18 19 20 21 22 23 (24) (25) 26 27 28 29 30 31 32 33 34+ // The_most_search_engines_use_only_one_of_these_methods._Even_the_search_engines_that_says_they_can_use_the_both_methods_basically+ // 12__13__(14)___(15)_____16__17___18__19_20____21_______22___23_(24)___(25)_____26___27___28___29__30__31__32___33______34- make1dmfIndex( longMVValues );+ make1dmfIndex(longMVValues);}-+protected void makeIndexLongMVB() throws Exception {// "*" ... LF-+// 1111111111222222222233333333334444444444555555// 01234567890123456789012345678901234567890123456789012345// *Lucene/Solr does not require such additional hardware.@@ -437,20 +429,20 @@ public abstract class AbstractTestCase extends LuceneTestCase {// lk 47 ce 55 ee 63// ed 64- make1dmfIndexB( biMVValues );+ make1dmfIndexB(biMVValues);}-+protected void makeIndexStrMV() throws Exception {// 0123// "abc"-+// 34567// "defg"// 111// 789012// "hijkl"- make1dmfIndexNA( strMVValues );+ make1dmfIndexNA(strMVValues);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestBreakIteratorBoundaryScanner.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestBreakIteratorBoundaryScanner.javaindex 272226c4742..6b138b95ec4 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestBreakIteratorBoundaryScanner.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestBreakIteratorBoundaryScanner.java@@ -18,20 +18,19 @@ package org.apache.lucene.search.vectorhighlight;import java.text.BreakIterator;import java.util.Locale;-import org.apache.lucene.util.LuceneTestCase;public class TestBreakIteratorBoundaryScanner extends LuceneTestCase {static final String TEXT =- "Apache Lucene(TM) is a high-performance, full-featured text search engine library written entirely in Java." +- "\nIt is a technology suitable for nearly any application that requires\n" +- "full-text search, especially cross-platform. \nApache Lucene is an open source project available for free download.";+ "Apache Lucene(TM) is a high-performance, full-featured text search engine library written entirely in Java."+ + "\nIt is a technology suitable for nearly any application that requires\n"+ + "full-text search, especially cross-platform. \nApache Lucene is an open source project available for free download.";public void testOutOfRange() throws Exception {StringBuilder text = new StringBuilder(TEXT);BreakIterator bi = BreakIterator.getWordInstance(Locale.ROOT);BoundaryScanner scanner = new BreakIteratorBoundaryScanner(bi);-+int start = TEXT.length() + 1;assertEquals(start, scanner.findStartOffset(text, start));assertEquals(start, scanner.findEndOffset(text, start));@@ -45,7 +44,7 @@ public class TestBreakIteratorBoundaryScanner extends LuceneTestCase {StringBuilder text = new StringBuilder(TEXT);BreakIterator bi = BreakIterator.getWordInstance(Locale.ROOT);BoundaryScanner scanner = new BreakIteratorBoundaryScanner(bi);-+int start = TEXT.indexOf("formance");int expected = TEXT.indexOf("high-performance");testFindStartOffset(text, start, expected, scanner);@@ -59,7 +58,7 @@ public class TestBreakIteratorBoundaryScanner extends LuceneTestCase {// we test this with default locale, it's randomized by LuceneTestCaseBreakIterator bi = BreakIterator.getSentenceInstance(Locale.getDefault());BoundaryScanner scanner = new BreakIteratorBoundaryScanner(bi);-+int start = TEXT.indexOf("any application");int expected = TEXT.indexOf("It is a");testFindStartOffset(text, start, expected, scanner);@@ -73,7 +72,7 @@ public class TestBreakIteratorBoundaryScanner extends LuceneTestCase {// we test this with default locale, it's randomized by LuceneTestCaseBreakIterator bi = BreakIterator.getLineInstance(Locale.getDefault());BoundaryScanner scanner = new BreakIteratorBoundaryScanner(bi);-+int start = TEXT.indexOf("any application");int expected = TEXT.indexOf("nearly");testFindStartOffset(text, start, expected, scanner);@@ -81,12 +80,14 @@ public class TestBreakIteratorBoundaryScanner extends LuceneTestCase {expected = TEXT.indexOf("application that requires");testFindEndOffset(text, start, expected, scanner);}-- private void testFindStartOffset(StringBuilder text, int start, int expected, BoundaryScanner scanner) throws Exception {++ private void testFindStartOffset(+ StringBuilder text, int start, int expected, BoundaryScanner scanner) throws Exception {assertEquals(expected, scanner.findStartOffset(text, start));}-- private void testFindEndOffset(StringBuilder text, int start, int expected, BoundaryScanner scanner) throws Exception {++ private void testFindEndOffset(+ StringBuilder text, int start, int expected, BoundaryScanner scanner) throws Exception {assertEquals(expected, scanner.findEndOffset(text, start));}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFastVectorHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFastVectorHighlighter.javaindex 7fbd8a3445f..ed29be2ed70 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFastVectorHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFastVectorHighlighter.java@@ -15,12 +15,12 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;+import java.io.IOException;import java.util.HashSet;import java.util.Map;import java.util.Set;import java.util.TreeMap;-import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.CannedTokenStream;import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;@@ -39,8 +39,8 @@ import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.Term;import org.apache.lucene.queries.CommonTermsQuery;import org.apache.lucene.queries.function.FunctionScoreQuery;-import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanClause;+import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.BoostQuery;import org.apache.lucene.search.DoubleValuesSource;@@ -57,10 +57,8 @@ import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.automaton.CharacterRunAutomaton;import org.apache.lucene.util.automaton.RegExp;-public class TestFastVectorHighlighter extends LuceneTestCase {--+public void testSimpleHighlightTest() throws IOException {Directory dir = newDirectory();IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));@@ -70,18 +68,24 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.setStoreTermVectorPositions(true);type.setStoreTermVectors(true);type.freeze();- Field field = new Field("field", "This is a test where foo is highlighed and should be highlighted", type);-+ Field field =+ new Field(+ "field", "This is a test where foo is highlighed and should be highlighted", type);+doc.add(field);writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();-+IndexReader reader = DirectoryReader.open(writer);int docId = 0;- FieldQuery fieldQuery = highlighter.getFieldQuery( new TermQuery(new Term("field", "foo")), reader );- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);- // highlighted results are centered- assertEquals("This is a test where <b>foo</b> is highlighed and should be highlighted", bestFragments[0]);+ FieldQuery fieldQuery =+ highlighter.getFieldQuery(new TermQuery(new Term("field", "foo")), reader);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);+ // highlighted results are centered+ assertEquals(+ "This is a test where <b>foo</b> is highlighed and should be highlighted",+ bestFragments[0]);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 52, 1);assertEquals("This is a test where <b>foo</b> is highlighed and should be", bestFragments[0]);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 30, 1);@@ -100,7 +104,9 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.setStoreTermVectorPositions(true);type.setStoreTermVectors(true);type.freeze();- Field field = new Field("field", "This is a test where foo is highlighed and should be highlighted", type);+ Field field =+ new Field(+ "field", "This is a test where foo is highlighed and should be highlighted", type);doc.add(field);writer.addDocument(doc);@@ -108,10 +114,17 @@ public class TestFastVectorHighlighter extends LuceneTestCase {IndexReader reader = DirectoryReader.open(writer);int docId = 0;- FieldQuery fieldQuery = highlighter.getFieldQuery( new FunctionScoreQuery(new TermQuery(new Term("field", "foo")), DoubleValuesSource.constant(1)), reader );- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);+ FieldQuery fieldQuery =+ highlighter.getFieldQuery(+ new FunctionScoreQuery(+ new TermQuery(new Term("field", "foo")), DoubleValuesSource.constant(1)),+ reader);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);// highlighted results are centered- assertEquals("This is a test where <b>foo</b> is highlighed and should be highlighted", bestFragments[0]);+ assertEquals(+ "This is a test where <b>foo</b> is highlighed and should be highlighted",+ bestFragments[0]);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 52, 1);assertEquals("This is a test where <b>foo</b> is highlighed and should be", bestFragments[0]);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 30, 1);@@ -120,7 +133,7 @@ public class TestFastVectorHighlighter extends LuceneTestCase {writer.close();dir.close();}-+public void testPhraseHighlightLongTextTest() throws IOException {Directory dir = newDirectory();IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));@@ -130,8 +143,11 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.setStoreTermVectorPositions(true);type.setStoreTermVectors(true);type.freeze();- Field text = new Field("text",- "Netscape was the general name for a series of web browsers originally produced by Netscape Communications Corporation, now a subsidiary of AOL The original browser was once the dominant browser in terms of usage share, but as a result of the first browser war it lost virtually all of its share to Internet Explorer Netscape was discontinued and support for all Netscape browsers and client products was terminated on March 1, 2008 Netscape Navigator was the name of Netscape\u0027s web browser from versions 1.0 through 4.8 The first beta release versions of the browser were released in 1994 and known as Mosaic and then Mosaic Netscape until a legal challenge from the National Center for Supercomputing Applications (makers of NCSA Mosaic, which many of Netscape\u0027s founders used to develop), led to the name change to Netscape Navigator The company\u0027s name also changed from Mosaic Communications Corporation to Netscape Communications Corporation The browser was easily the most advanced...", type);+ Field text =+ new Field(+ "text",+ "Netscape was the general name for a series of web browsers originally produced by Netscape Communications Corporation, now a subsidiary of AOL The original browser was once the dominant browser in terms of usage share, but as a result of the first browser war it lost virtually all of its share to Internet Explorer Netscape was discontinued and support for all Netscape browsers and client products was terminated on March 1, 2008 Netscape Navigator was the name of Netscape\u0027s web browser from versions 1.0 through 4.8 The first beta release versions of the browser were released in 1994 and known as Mosaic and then Mosaic Netscape until a legal challenge from the National Center for Supercomputing Applications (makers of NCSA Mosaic, which many of Netscape\u0027s founders used to develop), led to the name change to Netscape Navigator The company\u0027s name also changed from Mosaic Communications Corporation to Netscape Communications Corporation The browser was easily the most advanced...",+ type);doc.add(text);writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();@@ -143,27 +159,31 @@ public class TestFastVectorHighlighter extends LuceneTestCase {query.add(new TermQuery(new Term(field, "internet")), Occur.MUST);query.add(new TermQuery(new Term(field, "explorer")), Occur.MUST);FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 128, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 128, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);- assertEquals("first browser war it lost virtually all of its share to <b>Internet</b> <b>Explorer</b> Netscape was discontinued and support for all Netscape browsers", bestFragments[0]);+ assertEquals(+ "first browser war it lost virtually all of its share to <b>Internet</b> <b>Explorer</b> Netscape was discontinued and support for all Netscape browsers",+ bestFragments[0]);}-+{PhraseQuery query = new PhraseQuery(field, "internet", "explorer");FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 128, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 128, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);- assertEquals("first browser war it lost virtually all of its share to <b>Internet Explorer</b> Netscape was discontinued and support for all Netscape browsers", bestFragments[0]);+ assertEquals(+ "first browser war it lost virtually all of its share to <b>Internet Explorer</b> Netscape was discontinued and support for all Netscape browsers",+ bestFragments[0]);}reader.close();writer.close();dir.close();}-+// see LUCENE-4899public void testPhraseHighlightTest() throws IOException {Directory dir = newDirectory();@@ -174,8 +194,16 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.setStoreTermVectorPositions(true);type.setStoreTermVectors(true);type.freeze();- Field longTermField = new Field("long_term", "This is a test thisisaverylongwordandmakessurethisfails where foo is highlighed and should be highlighted", type);- Field noLongTermField = new Field("no_long_term", "This is a test where foo is highlighed and should be highlighted", type);+ Field longTermField =+ new Field(+ "long_term",+ "This is a test thisisaverylongwordandmakessurethisfails where foo is highlighed and should be highlighted",+ type);+ Field noLongTermField =+ new Field(+ "no_long_term",+ "This is a test where foo is highlighed and should be highlighted",+ type);doc.add(longTermField);doc.add(noLongTermField);@@ -190,8 +218,8 @@ public class TestFastVectorHighlighter extends LuceneTestCase {query.add(new TermQuery(new Term(field, "foo")), Occur.MUST);query.add(new TermQuery(new Term(field, "highlighed")), Occur.MUST);FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);assertEquals("<b>foo</b> is <b>highlighed</b> and", bestFragments[0]);@@ -203,36 +231,32 @@ public class TestFastVectorHighlighter extends LuceneTestCase {query.add(pq, Occur.MUST);query.add(new TermQuery(new Term(field, "highlighed")), Occur.MUST);FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);// highlighted results are centeredassertEquals(0, bestFragments.length);- bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 30, 1);+ bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, field, 30, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);assertEquals("a <b>test</b> where <b>foo</b> is <b>highlighed</b> and", bestFragments[0]);-}{PhraseQuery query = new PhraseQuery(3, field, "test", "foo", "highlighed");FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);// highlighted results are centeredassertEquals(0, bestFragments.length);- bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 30, 1);+ bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, field, 30, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);assertEquals("a <b>test</b> where <b>foo</b> is <b>highlighed</b> and", bestFragments[0]);-}{PhraseQuery query = new PhraseQuery(30, field, "test", "foo", "highlighed");FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);assertEquals(0, bestFragments.length);}{@@ -245,31 +269,29 @@ public class TestFastVectorHighlighter extends LuceneTestCase {query.add(pq, Occur.MUST);query.add(new TermQuery(new Term(field, "highlighed")), Occur.MUST);FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);assertEquals(0, bestFragments.length);-- bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 30, 1);++ bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, field, 30, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);assertEquals("a <b>test</b> where <b>foo</b> is <b>highlighed</b> and", bestFragments[0]);}-+field = "long_term";{BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add(new TermQuery(new Term(field,- "thisisaverylongwordandmakessurethisfails")), Occur.MUST);+ query.add(+ new TermQuery(new Term(field, "thisisaverylongwordandmakessurethisfails")), Occur.MUST);query.add(new TermQuery(new Term(field, "foo")), Occur.MUST);query.add(new TermQuery(new Term(field, "highlighed")), Occur.MUST);FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader,- docId, field, 18, 1);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, field, 18, 1);// highlighted results are centeredassertEquals(1, bestFragments.length);- assertEquals("<b>thisisaverylongwordandmakessurethisfails</b>",- bestFragments[0]);+ assertEquals("<b>thisisaverylongwordandmakessurethisfails</b>", bestFragments[0]);}reader.close();writer.close();@@ -278,44 +300,50 @@ public class TestFastVectorHighlighter extends LuceneTestCase {public void testBoostedPhraseHighlightTest() throws IOException {Directory dir = newDirectory();- IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(new MockAnalyzer( random() ) ) );+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));Document doc = new Document();- FieldType type = new FieldType( TextField.TYPE_STORED );- type.setStoreTermVectorOffsets( true );- type.setStoreTermVectorPositions( true );- type.setStoreTermVectors( true );+ FieldType type = new FieldType(TextField.TYPE_STORED);+ type.setStoreTermVectorOffsets(true);+ type.setStoreTermVectorPositions(true);+ type.setStoreTermVectors(true);type.freeze();StringBuilder text = new StringBuilder();- text.append("words words junk junk junk junk junk junk junk junk highlight junk junk junk junk together junk ");- for ( int i = 0; i<10; i++ ) {- text.append("junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk ");+ text.append(+ "words words junk junk junk junk junk junk junk junk highlight junk junk junk junk together junk ");+ for (int i = 0; i < 10; i++) {+ text.append(+ "junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk ");}text.append("highlight words together ");- for ( int i = 0; i<10; i++ ) {- text.append("junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk ");+ for (int i = 0; i < 10; i++) {+ text.append(+ "junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk junk ");}- doc.add( new Field( "text", text.toString().trim(), type ) );+ doc.add(new Field("text", text.toString().trim(), type));writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();IndexReader reader = DirectoryReader.open(writer);// This mimics what some query parsers do to <highlight words together>BooleanQuery.Builder terms = new BooleanQuery.Builder();- terms.add( clause( "text", "highlight" ), Occur.MUST );- terms.add( clause( "text", "words" ), Occur.MUST );- terms.add( clause( "text", "together" ), Occur.MUST );+ terms.add(clause("text", "highlight"), Occur.MUST);+ terms.add(clause("text", "words"), Occur.MUST);+ terms.add(clause("text", "together"), Occur.MUST);// This mimics what some query parsers do to <"highlight words together">BooleanQuery.Builder phraseB = new BooleanQuery.Builder();- phraseB.add( clause( "text", "highlight", "words", "together" ), Occur.MUST );+ phraseB.add(clause("text", "highlight", "words", "together"), Occur.MUST);Query phrase = phraseB.build();phrase = new BoostQuery(phrase, 100f);- // Now combine those results in a boolean query which should pull the phrases to the front of the list of fragments+ // Now combine those results in a boolean query which should pull the phrases to the front of+ // the list of fragmentsBooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( phrase, Occur.MUST );- query.add( phrase, Occur.SHOULD );- FieldQuery fieldQuery = new FieldQuery( query.build(), reader, true, false );- String fragment = highlighter.getBestFragment( fieldQuery, reader, 0, "text", 100 );- assertEquals( "junk junk junk junk junk junk junk junk <b>highlight words together</b> junk junk junk junk junk junk junk junk", fragment );+ query.add(phrase, Occur.MUST);+ query.add(phrase, Occur.SHOULD);+ FieldQuery fieldQuery = new FieldQuery(query.build(), reader, true, false);+ String fragment = highlighter.getBestFragment(fieldQuery, reader, 0, "text", 100);+ assertEquals(+ "junk junk junk junk junk junk junk junk <b>highlight words together</b> junk junk junk junk junk junk junk junk",+ fragment);reader.close();writer.close();@@ -324,20 +352,28 @@ public class TestFastVectorHighlighter extends LuceneTestCase {public void testCommonTermsQueryHighlight() throws IOException {Directory dir = newDirectory();- IndexWriter writer = new IndexWriter(dir,- newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))- .setMergePolicy(newLogMergePolicy())); // don't reorder doc ids+ IndexWriter writer =+ new IndexWriter(+ dir,+ newIndexWriterConfig(+ new MockAnalyzer(+ random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))+ .setMergePolicy(newLogMergePolicy())); // don't reorder doc idsFieldType type = new FieldType(TextField.TYPE_STORED);type.setStoreTermVectorOffsets(true);type.setStoreTermVectorPositions(true);type.setStoreTermVectors(true);type.freeze();String[] texts = {- "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",- "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",- "JFK has been shot", "John Kennedy has been shot",- "This text has a typo in referring to Keneddy",- "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };+ "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",+ "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",+ "JFK has been shot",+ "John Kennedy has been shot",+ "This text has a typo in referring to Keneddy",+ "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc",+ "y z x y z a b",+ "lets is a the lets is a the lets is a the lets"+ };for (int i = 0; i < texts.length; i++) {Document doc = new Document();Field field = new Field("field", texts[i], type);@@ -348,176 +384,281 @@ public class TestFastVectorHighlighter extends LuceneTestCase {query.add(new Term("field", "text"));query.add(new Term("field", "long"));query.add(new Term("field", "very"));-+FastVectorHighlighter highlighter = new FastVectorHighlighter();IndexReader reader = DirectoryReader.open(writer);IndexSearcher searcher = newSearcher(reader);TopDocs hits = searcher.search(query, 10);assertEquals(2, hits.totalHits.value);- FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);+ FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, 1, "field", 1000, 1);- assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);+ assertEquals(+ "This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy",+ bestFragments[0]);- fieldQuery = highlighter.getFieldQuery(query, reader);+ fieldQuery = highlighter.getFieldQuery(query, reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, 0, "field", 1000, 1);- assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);+ assertEquals(+ "Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot",+ bestFragments[0]);reader.close();writer.close();dir.close();}-+public void testMatchedFields() throws IOException {// Searching just on the stored field doesn't highlight a stopword- matchedFieldsTestCase( false, true, "a match", "a <b>match</b>",- clause( "field", "a" ), clause( "field", "match" ) );+ matchedFieldsTestCase(+ false, true, "a match", "a <b>match</b>", clause("field", "a"), clause("field", "match"));// Even if you add an unqueried matched field that would match it- matchedFieldsTestCase( "a match", "a <b>match</b>",- clause( "field", "a" ), clause( "field", "match" ) );+ matchedFieldsTestCase(+ "a match", "a <b>match</b>", clause("field", "a"), clause("field", "match"));// Nor if you query the field but don't add it as a matched field to the highlighter- matchedFieldsTestCase( false, false, "a match", "a <b>match</b>",- clause( "field_exact", "a" ), clause( "field", "match" ) );-- // But if you query the field and add it as a matched field to the highlighter then it is highlighted- matchedFieldsTestCase( "a match", "<b>a</b> <b>match</b>",- clause( "field_exact", "a" ), clause( "field", "match" ) );+ matchedFieldsTestCase(+ false,+ false,+ "a match",+ "a <b>match</b>",+ clause("field_exact", "a"),+ clause("field", "match"));++ // But if you query the field and add it as a matched field to the highlighter then it is+ // highlighted+ matchedFieldsTestCase(+ "a match", "<b>a</b> <b>match</b>", clause("field_exact", "a"), clause("field", "match"));// It is also ok to match just the matched field but get highlighting from the stored field- matchedFieldsTestCase( "a match", "<b>a</b> <b>match</b>",- clause( "field_exact", "a" ), clause( "field_exact", "match" ) );+ matchedFieldsTestCase(+ "a match",+ "<b>a</b> <b>match</b>",+ clause("field_exact", "a"),+ clause("field_exact", "match"));// Boosted matched fields work too- matchedFieldsTestCase( "a match", "<b>a</b> <b>match</b>",- clause( "field_exact", 5, "a" ), clause( "field", "match" ) );+ matchedFieldsTestCase(+ "a match",+ "<b>a</b> <b>match</b>",+ clause("field_exact", 5, "a"),+ clause("field", "match"));// It is also ok if both the stored and the matched field match the term- matchedFieldsTestCase( "a match", "a <b>match</b>",- clause( "field_exact", "match" ), clause( "field", "match" ) );+ matchedFieldsTestCase(+ "a match", "a <b>match</b>", clause("field_exact", "match"), clause("field", "match"));// And the highlighter respects the boosts on matched fields when sorting fragments- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_exact", 5, "a", "cat" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> <b>cat</b> junk junk junk junk",- clause( "field", "cat" ), clause( "field_exact", "a", "cat" ) );+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_exact", 5, "a", "cat"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> <b>cat</b> junk junk junk junk",+ clause("field", "cat"),+ clause("field_exact", "a", "cat"));// The same thing works across three fields as well- matchedFieldsTestCase( "cat cat CAT junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_exact", 200, "a", "cat" ), clause( "field_super_exact", 5, "CAT" ) );- matchedFieldsTestCase( "a cat cat junk junk junk junk junk junk junk a CAT junk junk",- "junk junk <b>a CAT</b> junk junk",- clause( "field", "cat" ), clause( "field_exact", 5, "a", "cat" ), clause( "field_super_exact", 200, "a", "CAT" ) );+ matchedFieldsTestCase(+ "cat cat CAT junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_exact", 200, "a", "cat"),+ clause("field_super_exact", 5, "CAT"));+ matchedFieldsTestCase(+ "a cat cat junk junk junk junk junk junk junk a CAT junk junk",+ "junk junk <b>a CAT</b> junk junk",+ clause("field", "cat"),+ clause("field_exact", 5, "a", "cat"),+ clause("field_super_exact", 200, "a", "CAT"));// And across fields with different tokenizers!- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field_exact", 5, "a", "cat" ), clause( "field_characters", "c" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>c</b>at <b>c</b>at junk junk junk junk",- clause( "field_exact", "a", "cat" ), clause( "field_characters", "c" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "ca<b>t</b> ca<b>t</b> junk junk junk junk",- clause( "field_exact", "a", "cat" ), clause( "field_characters", "t" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> <b>cat</b> junk junk junk junk", // See how the phrases are joined?- clause( "field", "cat" ), clause( "field_characters", 5, "c" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_characters", 5, "a", " ", "c", "a", "t" ) );+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field_exact", 5, "a", "cat"),+ clause("field_characters", "c"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>c</b>at <b>c</b>at junk junk junk junk",+ clause("field_exact", "a", "cat"),+ clause("field_characters", "c"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "ca<b>t</b> ca<b>t</b> junk junk junk junk",+ clause("field_exact", "a", "cat"),+ clause("field_characters", "t"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> <b>cat</b> junk junk junk junk", // See how the phrases are joined?+ clause("field", "cat"),+ clause("field_characters", 5, "c"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_characters", 5, "a", " ", "c", "a", "t"));// Phrases and tokens inside one another are joined- matchedFieldsTestCase( "cats wow", "<b>cats w</b>ow",- clause( "field", "cats" ), clause( "field_tripples", "s w" ) );+ matchedFieldsTestCase(+ "cats wow", "<b>cats w</b>ow", clause("field", "cats"), clause("field_tripples", "s w"));// Everything works pretty well even if you don't require a field match- matchedFieldsTestCase( true, false, "cat cat junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_characters", 10, "a", " ", "c", "a", "t" ) );+ matchedFieldsTestCase(+ true,+ false,+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_characters", 10, "a", " ", "c", "a", "t"));// Even boosts keep themselves pretty much intact- matchedFieldsTestCase( true, false, "a cat cat junk junk junk junk junk junk junk a CAT junk junk",- "junk junk <b>a CAT</b> junk junk",- clause( "field", "cat" ), clause( "field_exact", 5, "a", "cat" ), clause( "field_super_exact", 200, "a", "CAT" ) );- matchedFieldsTestCase( true, false, "cat cat CAT junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_exact", 200, "a", "cat" ), clause( "field_super_exact", 5, "CAT" ) );+ matchedFieldsTestCase(+ true,+ false,+ "a cat cat junk junk junk junk junk junk junk a CAT junk junk",+ "junk junk <b>a CAT</b> junk junk",+ clause("field", "cat"),+ clause("field_exact", 5, "a", "cat"),+ clause("field_super_exact", 200, "a", "CAT"));+ matchedFieldsTestCase(+ true,+ false,+ "cat cat CAT junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_exact", 200, "a", "cat"),+ clause("field_super_exact", 5, "CAT"));// Except that all the matched field matches apply even if they aren't mentioned in the query// which can make for some confusing scoring. This isn't too big a deal, just something you// need to think about when you don't force a field match.- matchedFieldsTestCase( true, false, "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> <b>cat</b> junk junk junk junk",- clause( "field", "cat" ), clause( "field_characters", 4, "a", " ", "c", "a", "t" ) );-- // It is also cool to match fields that don't have _exactly_ the same text so long as you are careful.+ matchedFieldsTestCase(+ true,+ false,+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> <b>cat</b> junk junk junk junk",+ clause("field", "cat"),+ clause("field_characters", 4, "a", " ", "c", "a", "t"));++ // It is also cool to match fields that don't have _exactly_ the same text so long as you are+ // careful.// In this case field_sliced is a prefix of field.- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> <b>cat</b> junk junk junk junk", clause( "field_sliced", "cat" ) );+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> <b>cat</b> junk junk junk junk",+ clause("field_sliced", "cat"));// Multiple matches add to the score of the segment- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> <b>cat</b> junk junk junk junk",- clause( "field", "cat" ), clause( "field_sliced", "cat" ), clause( "field_exact", 2, "a", "cat" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "junk junk <b>a cat</b> junk junk",- clause( "field", "cat" ), clause( "field_sliced", "cat" ), clause( "field_exact", 4, "a", "cat" ) );+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> <b>cat</b> junk junk junk junk",+ clause("field", "cat"),+ clause("field_sliced", "cat"),+ clause("field_exact", 2, "a", "cat"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "junk junk <b>a cat</b> junk junk",+ clause("field", "cat"),+ clause("field_sliced", "cat"),+ clause("field_exact", 4, "a", "cat"));// Even fields with tokens on top of one another are ok- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> cat junk junk junk junk",- clause( "field_der_red", 2, "der" ), clause( "field_exact", "a", "cat" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> cat junk junk junk junk",- clause( "field_der_red", 2, "red" ), clause( "field_exact", "a", "cat" ) );- matchedFieldsTestCase( "cat cat junk junk junk junk junk junk junk a cat junk junk",- "<b>cat</b> cat junk junk junk junk",- clause( "field_der_red", "red" ), clause( "field_der_red", "der" ), clause( "field_exact", "a", "cat" ) );+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> cat junk junk junk junk",+ clause("field_der_red", 2, "der"),+ clause("field_exact", "a", "cat"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> cat junk junk junk junk",+ clause("field_der_red", 2, "red"),+ clause("field_exact", "a", "cat"));+ matchedFieldsTestCase(+ "cat cat junk junk junk junk junk junk junk a cat junk junk",+ "<b>cat</b> cat junk junk junk junk",+ clause("field_der_red", "red"),+ clause("field_der_red", "der"),+ clause("field_exact", "a", "cat"));}public void testMultiValuedSortByScore() throws IOException {Directory dir = newDirectory();- IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(new MockAnalyzer( random() ) ) );+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));Document doc = new Document();- FieldType type = new FieldType( TextField.TYPE_STORED );- type.setStoreTermVectorOffsets( true );- type.setStoreTermVectorPositions( true );- type.setStoreTermVectors( true );+ FieldType type = new FieldType(TextField.TYPE_STORED);+ type.setStoreTermVectorOffsets(true);+ type.setStoreTermVectorPositions(true);+ type.setStoreTermVectors(true);type.freeze();- doc.add( new Field( "field", "zero if naught", type ) ); // The first two fields contain the best match- doc.add( new Field( "field", "hero of legend", type ) ); // but total a lower score (3) than the bottom- doc.add( new Field( "field", "naught of hero", type ) ); // two fields (4)- doc.add( new Field( "field", "naught of hero", type ) );+ doc.add(+ new Field("field", "zero if naught", type)); // The first two fields contain the best match+ doc.add(+ new Field("field", "hero of legend", type)); // but total a lower score (3) than the bottom+ doc.add(new Field("field", "naught of hero", type)); // two fields (4)+ doc.add(new Field("field", "naught of hero", type));writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();-- ScoreOrderFragmentsBuilder fragmentsBuilder = new ScoreOrderFragmentsBuilder();- fragmentsBuilder.setDiscreteMultiValueHighlighting( true );++ ScoreOrderFragmentsBuilder fragmentsBuilder = new ScoreOrderFragmentsBuilder();+ fragmentsBuilder.setDiscreteMultiValueHighlighting(true);IndexReader reader = DirectoryReader.open(writer);- String[] preTags = new String[] { "<b>" };- String[] postTags = new String[] { "</b>" };+ String[] preTags = new String[] {"<b>"};+ String[] postTags = new String[] {"</b>"};Encoder encoder = new DefaultEncoder();int docId = 0;BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( clause( "field", "hero" ), Occur.SHOULD);- query.add( clause( "field", "of" ), Occur.SHOULD);- query.add( clause( "field", "legend" ), Occur.SHOULD);- FieldQuery fieldQuery = highlighter.getFieldQuery( query.build(), reader );-- for ( FragListBuilder fragListBuilder : new FragListBuilder[] {- new SimpleFragListBuilder(), new WeightedFragListBuilder() } ) {- String[] bestFragments = highlighter.getBestFragments( fieldQuery, reader, docId, "field", 20, 1,- fragListBuilder, fragmentsBuilder, preTags, postTags, encoder );+ query.add(clause("field", "hero"), Occur.SHOULD);+ query.add(clause("field", "of"), Occur.SHOULD);+ query.add(clause("field", "legend"), Occur.SHOULD);+ FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);++ for (FragListBuilder fragListBuilder :+ new FragListBuilder[] {new SimpleFragListBuilder(), new WeightedFragListBuilder()}) {+ String[] bestFragments =+ highlighter.getBestFragments(+ fieldQuery,+ reader,+ docId,+ "field",+ 20,+ 1,+ fragListBuilder,+ fragmentsBuilder,+ preTags,+ postTags,+ encoder);assertEquals("<b>hero</b> <b>of</b> <b>legend</b>", bestFragments[0]);- bestFragments = highlighter.getBestFragments( fieldQuery, reader, docId, "field", 28, 1,- fragListBuilder, fragmentsBuilder, preTags, postTags, encoder );+ bestFragments =+ highlighter.getBestFragments(+ fieldQuery,+ reader,+ docId,+ "field",+ 28,+ 1,+ fragListBuilder,+ fragmentsBuilder,+ preTags,+ postTags,+ encoder);assertEquals("<b>hero</b> <b>of</b> <b>legend</b>", bestFragments[0]);- bestFragments = highlighter.getBestFragments( fieldQuery, reader, docId, "field", 30000, 1,- fragListBuilder, fragmentsBuilder, preTags, postTags, encoder );+ bestFragments =+ highlighter.getBestFragments(+ fieldQuery,+ reader,+ docId,+ "field",+ 30000,+ 1,+ fragListBuilder,+ fragmentsBuilder,+ preTags,+ postTags,+ encoder);assertEquals("<b>hero</b> <b>of</b> <b>legend</b>", bestFragments[0]);}@@ -536,7 +677,7 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.freeze();Document doc = new Document();- doc.add( new Field("field", "the quick brown fox", type ));+ doc.add(new Field("field", "the quick brown fox", type));writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();@@ -544,12 +685,14 @@ public class TestFastVectorHighlighter extends LuceneTestCase {int docId = 0;// query1: simple synonym query- SynonymQuery synQuery = new SynonymQuery.Builder("field")- .addTerm(new Term("field", "quick"))- .addTerm(new Term("field", "fast"))- .build();- FieldQuery fieldQuery = highlighter.getFieldQuery(synQuery, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);+ SynonymQuery synQuery =+ new SynonymQuery.Builder("field")+ .addTerm(new Term("field", "quick"))+ .addTerm(new Term("field", "fast"))+ .build();+ FieldQuery fieldQuery = highlighter.getFieldQuery(synQuery, reader);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);assertEquals("the <b>quick</b> brown fox", bestFragments[0]);// query2: boolean query with synonym query@@ -557,7 +700,7 @@ public class TestFastVectorHighlighter extends LuceneTestCase {new BooleanQuery.Builder().add(new BooleanClause(synQuery, Occur.MUST)).add(new BooleanClause(new TermQuery(new Term("field", "fox")), Occur.MUST));- fieldQuery = highlighter.getFieldQuery(bq.build(), reader);+ fieldQuery = highlighter.getFieldQuery(bq.build(), reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);assertEquals("the <b>quick</b> brown <b>fox</b>", bestFragments[0]);@@ -565,7 +708,7 @@ public class TestFastVectorHighlighter extends LuceneTestCase {writer.close();dir.close();}-+public void testBooleanPhraseWithSynonym() throws IOException {Directory dir = newDirectory();IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));@@ -577,43 +720,45 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.freeze();Token syn = new Token("httpwwwfacebookcom", 6, 29);syn.setPositionIncrement(0);- CannedTokenStream ts = new CannedTokenStream(- new Token("test", 0, 4),- new Token("http", 6, 10),- syn,- new Token("www", 13, 16),- new Token("facebook", 17, 25),- new Token("com", 26, 29)- );+ CannedTokenStream ts =+ new CannedTokenStream(+ new Token("test", 0, 4),+ new Token("http", 6, 10),+ syn,+ new Token("www", 13, 16),+ new Token("facebook", 17, 25),+ new Token("com", 26, 29));Field field = new Field("field", ts, type);doc.add(field);doc.add(new StoredField("field", "Test: http://www.facebook.com"));writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();-+IndexReader reader = DirectoryReader.open(writer);int docId = 0;-+// query1: matchPhraseQuery pq = new PhraseQuery("field", "test", "http", "www", "facebook", "com");- FieldQuery fieldQuery = highlighter.getFieldQuery(pq, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);+ FieldQuery fieldQuery = highlighter.getFieldQuery(pq, reader);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);-+// query2: match- PhraseQuery pq2 = new PhraseQuery("field", "test", "httpwwwfacebookcom", "www", "facebook", "com");- fieldQuery = highlighter.getFieldQuery(pq2, reader);+ PhraseQuery pq2 =+ new PhraseQuery("field", "test", "httpwwwfacebookcom", "www", "facebook", "com");+ fieldQuery = highlighter.getFieldQuery(pq2, reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);-+// query3: OR query1 and query2 togetherBooleanQuery.Builder bq = new BooleanQuery.Builder();bq.add(pq, BooleanClause.Occur.SHOULD);bq.add(pq2, BooleanClause.Occur.SHOULD);- fieldQuery = highlighter.getFieldQuery(bq.build(), reader);+ fieldQuery = highlighter.getFieldQuery(bq.build(), reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);-+reader.close();writer.close();dir.close();@@ -631,41 +776,46 @@ public class TestFastVectorHighlighter extends LuceneTestCase {type.freeze();Document doc = new Document();- doc.add( new Field( "field", "one two three five", type ) );- doc.add( new Field( "field", "two three four", type ) );- doc.add( new Field( "field", "five six five", type ) );- doc.add( new Field( "field", "six seven eight nine eight nine eight " +- "nine eight nine eight nine eight nine", type ) );- doc.add( new Field( "field", "eight nine", type ) );- doc.add( new Field( "field", "ten eleven", type ) );- doc.add( new Field( "field", "twelve thirteen", type ) );+ doc.add(new Field("field", "one two three five", type));+ doc.add(new Field("field", "two three four", type));+ doc.add(new Field("field", "five six five", type));+ doc.add(+ new Field(+ "field",+ "six seven eight nine eight nine eight " + "nine eight nine eight nine eight nine",+ type));+ doc.add(new Field("field", "eight nine", type));+ doc.add(new Field("field", "ten eleven", type));+ doc.add(new Field("field", "twelve thirteen", type));writer.addDocument(doc);BaseFragListBuilder fragListBuilder = new SimpleFragListBuilder();BaseFragmentsBuilder fragmentsBuilder = new SimpleFragmentsBuilder();fragmentsBuilder.setDiscreteMultiValueHighlighting(true);- FastVectorHighlighter highlighter = new FastVectorHighlighter(true, true, fragListBuilder, fragmentsBuilder);+ FastVectorHighlighter highlighter =+ new FastVectorHighlighter(true, true, fragListBuilder, fragmentsBuilder);IndexReader reader = DirectoryReader.open(writer);int docId = 0;// Phrase that spans a field valueQuery q = new PhraseQuery("field", "four", "five");- FieldQuery fieldQuery = highlighter.getFieldQuery(q, reader);- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);+ FieldQuery fieldQuery = highlighter.getFieldQuery(q, reader);+ String[] bestFragments =+ highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);assertEquals("two three <b>four</b>", bestFragments[0]);assertEquals("<b>five</b> six five", bestFragments[1]);assertEquals(2, bestFragments.length);// Phrase that ends at a field valueq = new PhraseQuery("field", "three", "five");- fieldQuery = highlighter.getFieldQuery(q, reader);+ fieldQuery = highlighter.getFieldQuery(q, reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);assertEquals("one two <b>three five</b>", bestFragments[0]);assertEquals(1, bestFragments.length);// Phrase that spans across three valuesq = new PhraseQuery("field", "nine", "ten", "eleven", "twelve");- fieldQuery = highlighter.getFieldQuery(q, reader);+ fieldQuery = highlighter.getFieldQuery(q, reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);assertEquals("eight <b>nine</b>", bestFragments[0]);assertEquals("<b>ten eleven</b>", bestFragments[1]);@@ -674,7 +824,7 @@ public class TestFastVectorHighlighter extends LuceneTestCase {// Term query that appears in multiple valuesq = new TermQuery(new Term("field", "two"));- fieldQuery = highlighter.getFieldQuery(q, reader);+ fieldQuery = highlighter.getFieldQuery(q, reader);bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);assertEquals("one <b>two</b> three five", bestFragments[0]);assertEquals("<b>two</b> three four", bestFragments[1]);@@ -685,106 +835,158 @@ public class TestFastVectorHighlighter extends LuceneTestCase {dir.close();}- private void matchedFieldsTestCase( String fieldValue, String expected, Query... queryClauses ) throws IOException {- matchedFieldsTestCase( true, true, fieldValue, expected, queryClauses );+ private void matchedFieldsTestCase(String fieldValue, String expected, Query... queryClauses)+ throws IOException {+ matchedFieldsTestCase(true, true, fieldValue, expected, queryClauses);}- private void matchedFieldsTestCase( boolean useMatchedFields, boolean fieldMatch, String fieldValue, String expected, Query... queryClauses ) throws IOException {+ private void matchedFieldsTestCase(+ boolean useMatchedFields,+ boolean fieldMatch,+ String fieldValue,+ String expected,+ Query... queryClauses)+ throws IOException {Document doc = new Document();- FieldType stored = new FieldType( TextField.TYPE_STORED );- stored.setStoreTermVectorOffsets( true );- stored.setStoreTermVectorPositions( true );- stored.setStoreTermVectors( true );+ FieldType stored = new FieldType(TextField.TYPE_STORED);+ stored.setStoreTermVectorOffsets(true);+ stored.setStoreTermVectorPositions(true);+ stored.setStoreTermVectors(true);stored.freeze();- FieldType matched = new FieldType( TextField.TYPE_NOT_STORED );- matched.setStoreTermVectorOffsets( true );- matched.setStoreTermVectorPositions( true );- matched.setStoreTermVectors( true );+ FieldType matched = new FieldType(TextField.TYPE_NOT_STORED);+ matched.setStoreTermVectorOffsets(true);+ matched.setStoreTermVectorPositions(true);+ matched.setStoreTermVectors(true);matched.freeze();- doc.add( new Field( "field", fieldValue, stored ) ); // Whitespace tokenized with English stop words- doc.add( new Field( "field_exact", fieldValue, matched ) ); // Whitespace tokenized without stop words- doc.add( new Field( "field_super_exact", fieldValue, matched ) ); // Whitespace tokenized without toLower- doc.add( new Field( "field_characters", fieldValue, matched ) ); // Each letter is a token- doc.add( new Field( "field_tripples", fieldValue, matched ) ); // Every three letters is a token- doc.add( new Field( "field_sliced", fieldValue.substring( 0, // Sliced at 10 chars then analyzed just like field- Math.min( fieldValue.length() - 1 , 10 ) ), matched ) );- doc.add( new Field( "field_der_red", new CannedTokenStream( // Hacky field containing "der" and "red" at pos = 0- token( "der", 1, 0, 3 ),- token( "red", 0, 0, 3 )- ), matched ) );+ doc.add(new Field("field", fieldValue, stored)); // Whitespace tokenized with English stop words+ doc.add(+ new Field("field_exact", fieldValue, matched)); // Whitespace tokenized without stop words+ doc.add(+ new Field(+ "field_super_exact", fieldValue, matched)); // Whitespace tokenized without toLower+ doc.add(new Field("field_characters", fieldValue, matched)); // Each letter is a token+ doc.add(new Field("field_tripples", fieldValue, matched)); // Every three letters is a token+ doc.add(+ new Field(+ "field_sliced",+ fieldValue.substring(+ 0, // Sliced at 10 chars then analyzed just like field+ Math.min(fieldValue.length() - 1, 10)),+ matched));+ doc.add(+ new Field(+ "field_der_red",+ new CannedTokenStream( // Hacky field containing "der" and "red" at pos = 0+ token("der", 1, 0, 3), token("red", 0, 0, 3)),+ matched));final Map<String, Analyzer> fieldAnalyzers = new TreeMap<>();- fieldAnalyzers.put( "field", new MockAnalyzer( random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET ) );- fieldAnalyzers.put( "field_exact", new MockAnalyzer( random() ) );- fieldAnalyzers.put( "field_super_exact", new MockAnalyzer( random(), MockTokenizer.WHITESPACE, false ) );- fieldAnalyzers.put( "field_characters", new MockAnalyzer( random(), new CharacterRunAutomaton( new RegExp(".").toAutomaton() ), true ) );- fieldAnalyzers.put( "field_tripples", new MockAnalyzer( random(), new CharacterRunAutomaton( new RegExp("...").toAutomaton() ), true ) );- fieldAnalyzers.put( "field_sliced", fieldAnalyzers.get( "field" ) );- fieldAnalyzers.put( "field_der_red", fieldAnalyzers.get( "field" ) ); // This is required even though we provide a token stream- Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {- public Analyzer getWrappedAnalyzer(String fieldName) {- return fieldAnalyzers.get( fieldName );- }- };+ fieldAnalyzers.put(+ "field",+ new MockAnalyzer(+ random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET));+ fieldAnalyzers.put("field_exact", new MockAnalyzer(random()));+ fieldAnalyzers.put(+ "field_super_exact", new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));+ fieldAnalyzers.put(+ "field_characters",+ new MockAnalyzer(random(), new CharacterRunAutomaton(new RegExp(".").toAutomaton()), true));+ fieldAnalyzers.put(+ "field_tripples",+ new MockAnalyzer(+ random(), new CharacterRunAutomaton(new RegExp("...").toAutomaton()), true));+ fieldAnalyzers.put("field_sliced", fieldAnalyzers.get("field"));+ fieldAnalyzers.put(+ "field_der_red",+ fieldAnalyzers.get("field")); // This is required even though we provide a token stream+ Analyzer analyzer =+ new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {+ public Analyzer getWrappedAnalyzer(String fieldName) {+ return fieldAnalyzers.get(fieldName);+ }+ };Directory dir = newDirectory();- IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(analyzer));- writer.addDocument( doc );+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(analyzer));+ writer.addDocument(doc);FastVectorHighlighter highlighter = new FastVectorHighlighter();FragListBuilder fragListBuilder = new SimpleFragListBuilder();FragmentsBuilder fragmentsBuilder = new ScoreOrderFragmentsBuilder();IndexReader reader = DirectoryReader.open(writer);- String[] preTags = new String[] { "<b>" };- String[] postTags = new String[] { "</b>" };+ String[] preTags = new String[] {"<b>"};+ String[] postTags = new String[] {"</b>"};Encoder encoder = new DefaultEncoder();int docId = 0;BooleanQuery.Builder query = new BooleanQuery.Builder();- for ( Query clause : queryClauses ) {- query.add( clause, Occur.MUST );+ for (Query clause : queryClauses) {+ query.add(clause, Occur.MUST);}- FieldQuery fieldQuery = new FieldQuery( query.build(), reader, true, fieldMatch );+ FieldQuery fieldQuery = new FieldQuery(query.build(), reader, true, fieldMatch);String[] bestFragments;- if ( useMatchedFields ) {- Set< String > matchedFields = new HashSet<>();- matchedFields.add( "field" );- matchedFields.add( "field_exact" );- matchedFields.add( "field_super_exact" );- matchedFields.add( "field_characters" );- matchedFields.add( "field_tripples" );- matchedFields.add( "field_sliced" );- matchedFields.add( "field_der_red" );- bestFragments = highlighter.getBestFragments( fieldQuery, reader, docId, "field", matchedFields, 25, 1,- fragListBuilder, fragmentsBuilder, preTags, postTags, encoder );+ if (useMatchedFields) {+ Set<String> matchedFields = new HashSet<>();+ matchedFields.add("field");+ matchedFields.add("field_exact");+ matchedFields.add("field_super_exact");+ matchedFields.add("field_characters");+ matchedFields.add("field_tripples");+ matchedFields.add("field_sliced");+ matchedFields.add("field_der_red");+ bestFragments =+ highlighter.getBestFragments(+ fieldQuery,+ reader,+ docId,+ "field",+ matchedFields,+ 25,+ 1,+ fragListBuilder,+ fragmentsBuilder,+ preTags,+ postTags,+ encoder);} else {- bestFragments = highlighter.getBestFragments( fieldQuery, reader, docId, "field", 25, 1,- fragListBuilder, fragmentsBuilder, preTags, postTags, encoder );+ bestFragments =+ highlighter.getBestFragments(+ fieldQuery,+ reader,+ docId,+ "field",+ 25,+ 1,+ fragListBuilder,+ fragmentsBuilder,+ preTags,+ postTags,+ encoder);}- assertEquals( expected, bestFragments[ 0 ] );+ assertEquals(expected, bestFragments[0]);reader.close();writer.close();dir.close();}- private Query clause( String field, String... terms ) {- return clause( field, 1, terms );+ private Query clause(String field, String... terms) {+ return clause(field, 1, terms);}- private Query clause( String field, float boost, String... terms ) {+ private Query clause(String field, float boost, String... terms) {Query q;- if ( terms.length == 1 ) {- q = new TermQuery( new Term( field, terms[ 0 ] ) );+ if (terms.length == 1) {+ q = new TermQuery(new Term(field, terms[0]));} else {q = new PhraseQuery(field, terms);}- q = new BoostQuery( q, boost );+ q = new BoostQuery(q, boost);return q;}- private static Token token( String term, int posInc, int startOffset, int endOffset ) {- Token t = new Token( term, startOffset, endOffset );- t.setPositionIncrement( posInc );+ private static Token token(String term, int posInc, int startOffset, int endOffset) {+ Token t = new Token(term, startOffset, endOffset);+ t.setPositionIncrement(posInc);return t;}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldPhraseList.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldPhraseList.javaindex 3458f2b5785..f96f5c1990c 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldPhraseList.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldPhraseList.java@@ -15,186 +15,188 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;-import java.util.LinkedList;-import org.apache.lucene.search.BooleanQuery;+import java.util.LinkedList;import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;import org.apache.lucene.util.TestUtil;public class TestFieldPhraseList extends AbstractTestCase {-+public void test1TermIndex() throws Exception {- make1d1fIndex( "a" );-- FieldQuery fq = new FieldQuery( tq( "a" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );-- fq = new FieldQuery( tq( "b" ), true, true );- stack = new FieldTermStack( reader, 0, F, fq );- fpl = new FieldPhraseList( stack, fq );- assertEquals( 0, fpl.phraseList.size() );+ make1d1fIndex("a");++ FieldQuery fq = new FieldQuery(tq("a"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("a(1.0)((0,1))", fpl.phraseList.get(0).toString());++ fq = new FieldQuery(tq("b"), true, true);+ stack = new FieldTermStack(reader, 0, F, fq);+ fpl = new FieldPhraseList(stack, fq);+ assertEquals(0, fpl.phraseList.size());}-+public void test2TermsIndex() throws Exception {- make1d1fIndex( "a a" );-- FieldQuery fq = new FieldQuery( tq( "a" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 2, fpl.phraseList.size() );- assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );- assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 1 ).toString() );+ make1d1fIndex("a a");++ FieldQuery fq = new FieldQuery(tq("a"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(2, fpl.phraseList.size());+ assertEquals("a(1.0)((0,1))", fpl.phraseList.get(0).toString());+ assertEquals("a(1.0)((2,3))", fpl.phraseList.get(1).toString());}-+public void test1PhraseIndex() throws Exception {- make1d1fIndex( "a b" );-- FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "ab(1.0)((0,3))", fpl.phraseList.get( 0 ).toString() );-- fq = new FieldQuery( tq( "b" ), true, true );- stack = new FieldTermStack( reader, 0, F, fq );- fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "b(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() );+ make1d1fIndex("a b");++ FieldQuery fq = new FieldQuery(pqF("a", "b"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("ab(1.0)((0,3))", fpl.phraseList.get(0).toString());++ fq = new FieldQuery(tq("b"), true, true);+ stack = new FieldTermStack(reader, 0, F, fq);+ fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("b(1.0)((2,3))", fpl.phraseList.get(0).toString());}-+public void test1PhraseIndexB() throws Exception {// 01 12 23 34 45 56 67 78 (offsets)// bb|bb|ba|ac|cb|ba|ab|bc// 0 1 2 3 4 5 6 7 (positions)- make1d1fIndexB( "bbbacbabc" );+ make1d1fIndexB("bbbacbabc");- FieldQuery fq = new FieldQuery( pqF( "ba", "ac" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "baac(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() );+ FieldQuery fq = new FieldQuery(pqF("ba", "ac"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("baac(1.0)((2,5))", fpl.phraseList.get(0).toString());}-+public void test2ConcatTermsIndexB() throws Exception {// 01 12 23 (offsets)// ab|ba|ab// 0 1 2 (positions)- make1d1fIndexB( "abab" );-- FieldQuery fq = new FieldQuery( tq( "ab" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 2, fpl.phraseList.size() );- assertEquals( "ab(1.0)((0,2))", fpl.phraseList.get( 0 ).toString() );- assertEquals( "ab(1.0)((2,4))", fpl.phraseList.get( 1 ).toString() );+ make1d1fIndexB("abab");++ FieldQuery fq = new FieldQuery(tq("ab"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(2, fpl.phraseList.size());+ assertEquals("ab(1.0)((0,2))", fpl.phraseList.get(0).toString());+ assertEquals("ab(1.0)((2,4))", fpl.phraseList.get(1).toString());}-+public void test2Terms1PhraseIndex() throws Exception {- make1d1fIndex( "c a a b" );+ make1d1fIndex("c a a b");// phraseHighlight = true- FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 0 ).toString() );+ FieldQuery fq = new FieldQuery(pqF("a", "b"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("ab(1.0)((4,7))", fpl.phraseList.get(0).toString());// phraseHighlight = false- fq = new FieldQuery( pqF( "a", "b" ), false, true );- stack = new FieldTermStack( reader, 0, F, fq );- fpl = new FieldPhraseList( stack, fq );- assertEquals( 2, fpl.phraseList.size() );- assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() );- assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 1 ).toString() );+ fq = new FieldQuery(pqF("a", "b"), false, true);+ stack = new FieldTermStack(reader, 0, F, fq);+ fpl = new FieldPhraseList(stack, fq);+ assertEquals(2, fpl.phraseList.size());+ assertEquals("a(1.0)((2,3))", fpl.phraseList.get(0).toString());+ assertEquals("ab(1.0)((4,7))", fpl.phraseList.get(1).toString());}-+public void testPhraseSlop() throws Exception {- make1d1fIndex( "c a a b c" );-- FieldQuery fq = new FieldQuery( pqF( 2F, 1, "a", "c" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "ac(2.0)((4,5)(8,9))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 9, fpl.phraseList.get( 0 ).getEndOffset() );+ make1d1fIndex("c a a b c");++ FieldQuery fq = new FieldQuery(pqF(2F, 1, "a", "c"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("ac(2.0)((4,5)(8,9))", fpl.phraseList.get(0).toString());+ assertEquals(4, fpl.phraseList.get(0).getStartOffset());+ assertEquals(9, fpl.phraseList.get(0).getEndOffset());}-+public void test2PhrasesOverlap() throws Exception {- make1d1fIndex( "d a b c d" );+ make1d1fIndex("d a b c d");BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "b" ), Occur.SHOULD );- query.add( pqF( "b", "c" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( query.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "abc(1.0)((2,7))", fpl.phraseList.get( 0 ).toString() );+ query.add(pqF("a", "b"), Occur.SHOULD);+ query.add(pqF("b", "c"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(query.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("abc(1.0)((2,7))", fpl.phraseList.get(0).toString());}-+public void test3TermsPhrase() throws Exception {- make1d1fIndex( "d a b a b c d" );+ make1d1fIndex("d a b a b c d");- FieldQuery fq = new FieldQuery( pqF( "a", "b", "c" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "abc(1.0)((6,11))", fpl.phraseList.get( 0 ).toString() );+ FieldQuery fq = new FieldQuery(pqF("a", "b", "c"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("abc(1.0)((6,11))", fpl.phraseList.get(0).toString());}-+public void testSearchLongestPhrase() throws Exception {- make1d1fIndex( "d a b d c a b c" );+ make1d1fIndex("d a b d c a b c");BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "b" ), Occur.SHOULD );- query.add( pqF( "a", "b", "c" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( query.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 2, fpl.phraseList.size() );- assertEquals( "ab(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() );- assertEquals( "abc(1.0)((10,15))", fpl.phraseList.get( 1 ).toString() );+ query.add(pqF("a", "b"), Occur.SHOULD);+ query.add(pqF("a", "b", "c"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(query.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(2, fpl.phraseList.size());+ assertEquals("ab(1.0)((2,5))", fpl.phraseList.get(0).toString());+ assertEquals("abc(1.0)((10,15))", fpl.phraseList.get(1).toString());}-+public void test1PhraseShortMV() throws Exception {makeIndexShortMV();- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "d(1.0)((9,10))", fpl.phraseList.get( 0 ).toString() );+ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("d(1.0)((9,10))", fpl.phraseList.get(0).toString());}-+public void test1PhraseLongMV() throws Exception {makeIndexLongMV();- FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 2, fpl.phraseList.size() );- assertEquals( "searchengines(1.0)((102,116))", fpl.phraseList.get( 0 ).toString() );- assertEquals( "searchengines(1.0)((157,171))", fpl.phraseList.get( 1 ).toString() );+ FieldQuery fq = new FieldQuery(pqF("search", "engines"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(2, fpl.phraseList.size());+ assertEquals("searchengines(1.0)((102,116))", fpl.phraseList.get(0).toString());+ assertEquals("searchengines(1.0)((157,171))", fpl.phraseList.get(1).toString());}public void test1PhraseLongMVB() throws Exception {makeIndexLongMVB();- FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "sppeeeed(1.0)((88,93))", fpl.phraseList.get( 0 ).toString() );+ FieldQuery fq =+ new FieldQuery(+ pqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("sppeeeed(1.0)((88,93))", fpl.phraseList.get(0).toString());}- /* This test shows a big speedup from limiting the number of analyzed phrases in+ /* This test shows a big speedup from limiting the number of analyzed phrases in* this bad case for FieldPhraseList *//* But it is not reliable as a unit test since it is timing-dependentpublic void testManyRepeatedTerms() throws Exception {@@ -220,71 +222,71 @@ public class TestFieldPhraseList extends AbstractTestCase {assertEquals( 16000, fpl.phraseList.size() );elseassertEquals( limit, fpl.phraseList.size() );- assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );+ assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() );}*/public void testWeightedPhraseInfoComparisonConsistency() {- WeightedPhraseInfo a = newInfo( 0, 0, 1 );- WeightedPhraseInfo b = newInfo( 1, 2, 1 );- WeightedPhraseInfo c = newInfo( 2, 3, 1 );- WeightedPhraseInfo d = newInfo( 0, 0, 1 );- WeightedPhraseInfo e = newInfo( 0, 0, 2 );-- assertConsistentEquals( a, a );- assertConsistentEquals( b, b );- assertConsistentEquals( c, c );- assertConsistentEquals( d, d );- assertConsistentEquals( e, e );- assertConsistentEquals( a, d );- assertConsistentLessThan( a, b );- assertConsistentLessThan( b, c );- assertConsistentLessThan( a, c );- assertConsistentLessThan( a, e );- assertConsistentLessThan( e, b );- assertConsistentLessThan( e, c );- assertConsistentLessThan( d, b );- assertConsistentLessThan( d, c );- assertConsistentLessThan( d, e );+ WeightedPhraseInfo a = newInfo(0, 0, 1);+ WeightedPhraseInfo b = newInfo(1, 2, 1);+ WeightedPhraseInfo c = newInfo(2, 3, 1);+ WeightedPhraseInfo d = newInfo(0, 0, 1);+ WeightedPhraseInfo e = newInfo(0, 0, 2);++ assertConsistentEquals(a, a);+ assertConsistentEquals(b, b);+ assertConsistentEquals(c, c);+ assertConsistentEquals(d, d);+ assertConsistentEquals(e, e);+ assertConsistentEquals(a, d);+ assertConsistentLessThan(a, b);+ assertConsistentLessThan(b, c);+ assertConsistentLessThan(a, c);+ assertConsistentLessThan(a, e);+ assertConsistentLessThan(e, b);+ assertConsistentLessThan(e, c);+ assertConsistentLessThan(d, b);+ assertConsistentLessThan(d, c);+ assertConsistentLessThan(d, e);}public void testToffsComparisonConsistency() {- Toffs a = new Toffs( 0, 0 );- Toffs b = new Toffs( 1, 2 );- Toffs c = new Toffs( 2, 3 );- Toffs d = new Toffs( 0, 0 );-- assertConsistentEquals( a, a );- assertConsistentEquals( b, b );- assertConsistentEquals( c, c );- assertConsistentEquals( d, d );- assertConsistentEquals( a, d );- assertConsistentLessThan( a, b );- assertConsistentLessThan( b, c );- assertConsistentLessThan( a, c );- assertConsistentLessThan( d, b );- assertConsistentLessThan( d, c );+ Toffs a = new Toffs(0, 0);+ Toffs b = new Toffs(1, 2);+ Toffs c = new Toffs(2, 3);+ Toffs d = new Toffs(0, 0);++ assertConsistentEquals(a, a);+ assertConsistentEquals(b, b);+ assertConsistentEquals(c, c);+ assertConsistentEquals(d, d);+ assertConsistentEquals(a, d);+ assertConsistentLessThan(a, b);+ assertConsistentLessThan(b, c);+ assertConsistentLessThan(a, c);+ assertConsistentLessThan(d, b);+ assertConsistentLessThan(d, c);}- private WeightedPhraseInfo newInfo( int startOffset, int endOffset, float boost ) {- LinkedList< TermInfo > infos = new LinkedList<>();- infos.add( new TermInfo( TestUtil.randomUnicodeString(random()), startOffset, endOffset, 0, 0 ) );- return new WeightedPhraseInfo( infos, boost );+ private WeightedPhraseInfo newInfo(int startOffset, int endOffset, float boost) {+ LinkedList<TermInfo> infos = new LinkedList<>();+ infos.add(new TermInfo(TestUtil.randomUnicodeString(random()), startOffset, endOffset, 0, 0));+ return new WeightedPhraseInfo(infos, boost);}- private < T extends Comparable< T > > void assertConsistentEquals( T a, T b ) {- assertEquals( a, b );- assertEquals( b, a );- assertEquals( a.hashCode(), b.hashCode() );- assertEquals( 0, a.compareTo( b ) );- assertEquals( 0, b.compareTo( a ) );+ private <T extends Comparable<T>> void assertConsistentEquals(T a, T b) {+ assertEquals(a, b);+ assertEquals(b, a);+ assertEquals(a.hashCode(), b.hashCode());+ assertEquals(0, a.compareTo(b));+ assertEquals(0, b.compareTo(a));}- private < T extends Comparable< T > > void assertConsistentLessThan( T a, T b ) {- assertFalse( a.equals( b ) );- assertFalse( b.equals( a ) );- assertFalse( a.hashCode() == b.hashCode() );- assertTrue( a.compareTo( b ) < 0 );- assertTrue( b.compareTo( a ) > 0 );+ private <T extends Comparable<T>> void assertConsistentLessThan(T a, T b) {+ assertFalse(a.equals(b));+ assertFalse(b.equals(a));+ assertFalse(a.hashCode() == b.hashCode());+ assertTrue(a.compareTo(b) < 0);+ assertTrue(b.compareTo(a) > 0);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldQuery.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldQuery.javaindex 2f9696188ad..42fec6dba55 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldQuery.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldQuery.java@@ -22,7 +22,6 @@ import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;-import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;@@ -42,9 +41,7 @@ import org.apache.lucene.util.BytesRef;public class TestFieldQuery extends AbstractTestCase {private float boost;- /**- * Set boost to a random value each time it is called.- */+ /** Set boost to a random value each time it is called. */private void initBoost() {boost = usually() ? 1F : random().nextFloat() * 10000;}@@ -64,20 +61,20 @@ public class TestFieldQuery extends AbstractTestCase {Query booleanQuery = booleanQueryB.build();booleanQuery = new BoostQuery(booleanQuery, boost);- FieldQuery fq = new FieldQuery(booleanQuery, true, true );+ FieldQuery fq = new FieldQuery(booleanQuery, true, true);Set<Query> flatQueries = new HashSet<>();fq.flatten(booleanQuery, reader, flatQueries, 1f);- assertCollectionQueries( flatQueries, tq( boost, "A" ), tq( boost, "B" ), tq( boost, "C" ) );+ assertCollectionQueries(flatQueries, tq(boost, "A"), tq(boost, "B"), tq(boost, "C"));}public void testFlattenDisjunctionMaxQuery() throws Exception {initBoost();- Query query = dmq( tq( "A" ), tq( "B" ), pqF( "C", "D" ) );- query = new BoostQuery( query, boost );- FieldQuery fq = new FieldQuery( query, true, true );+ Query query = dmq(tq("A"), tq("B"), pqF("C", "D"));+ query = new BoostQuery(query, boost);+ FieldQuery fq = new FieldQuery(query, true, true);Set<Query> flatQueries = new HashSet<>();- fq.flatten( query, reader, flatQueries, 1f );- assertCollectionQueries( flatQueries, tq( boost, "A" ), tq( boost, "B" ), pqF( boost, "C", "D" ) );+ fq.flatten(query, reader, flatQueries, 1f);+ assertCollectionQueries(flatQueries, tq(boost, "A"), tq(boost, "B"), pqF(boost, "C", "D"));}public void testFlattenTermAndPhrase() throws Exception {@@ -88,10 +85,10 @@ public class TestFieldQuery extends AbstractTestCase {Query booleanQuery = booleanQueryB.build();booleanQuery = new BoostQuery(booleanQuery, boost);- FieldQuery fq = new FieldQuery(booleanQuery, true, true );+ FieldQuery fq = new FieldQuery(booleanQuery, true, true);Set<Query> flatQueries = new HashSet<>();fq.flatten(booleanQuery, reader, flatQueries, 1f);- assertCollectionQueries( flatQueries, tq( boost, "A" ), pqF( boost, "B", "C" ) );+ assertCollectionQueries(flatQueries, tq(boost, "A"), pqF(boost, "B", "C"));}public void testFlattenTermAndPhrase2gram() throws Exception {@@ -100,179 +97,178 @@ public class TestFieldQuery extends AbstractTestCase {query.add(toPhraseQuery(analyze("BCD", F, analyzerB), F), Occur.MUST);query.add(toPhraseQuery(analyze("EFGH", F, analyzerB), F), Occur.SHOULD);- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Set<Query> flatQueries = new HashSet<>();- fq.flatten( query.build(), reader, flatQueries, 1f );- assertCollectionQueries( flatQueries, tq( "AA" ), pqF( "BC", "CD" ), pqF( "EF", "FG", "GH" ) );+ fq.flatten(query.build(), reader, flatQueries, 1f);+ assertCollectionQueries(flatQueries, tq("AA"), pqF("BC", "CD"), pqF("EF", "FG", "GH"));}public void testFlatten1TermPhrase() throws Exception {- Query query = pqF( "A" );- FieldQuery fq = new FieldQuery( query, true, true );+ Query query = pqF("A");+ FieldQuery fq = new FieldQuery(query, true, true);Set<Query> flatQueries = new HashSet<>();- fq.flatten( query, reader, flatQueries, 1f );- assertCollectionQueries( flatQueries, tq( "A" ) );+ fq.flatten(query, reader, flatQueries, 1f);+ assertCollectionQueries(flatQueries, tq("A"));}public void testExpand() throws Exception {- Query dummy = pqF( "DUMMY" );- FieldQuery fq = new FieldQuery( dummy, true, true );+ Query dummy = pqF("DUMMY");+ FieldQuery fq = new FieldQuery(dummy, true, true);// "a b","b c" => "a b","b c","a b c"Set<Query> flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "b", "c" ), pqF( "a", "b", "c" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("b", "c"));+ assertCollectionQueries(+ fq.expand(flatQueries), pqF("a", "b"), pqF("b", "c"), pqF("a", "b", "c"));// "a b","b c d" => "a b","b c d","a b c d"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "b", "c", "d" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("b", "c", "d"));+ assertCollectionQueries(+ fq.expand(flatQueries), pqF("a", "b"), pqF("b", "c", "d"), pqF("a", "b", "c", "d"));// "a b c","b c d" => "a b c","b c d","a b c d"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "c" ) );- flatQueries.add( pqF( "b", "c", "d" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "c" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) );+ flatQueries.add(pqF("a", "b", "c"));+ flatQueries.add(pqF("b", "c", "d"));+ assertCollectionQueries(+ fq.expand(flatQueries), pqF("a", "b", "c"), pqF("b", "c", "d"), pqF("a", "b", "c", "d"));// "a b c","c d e" => "a b c","c d e","a b c d e"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "c" ) );- flatQueries.add( pqF( "c", "d", "e" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "c" ), pqF( "c", "d", "e" ), pqF( "a", "b", "c", "d", "e" ) );+ flatQueries.add(pqF("a", "b", "c"));+ flatQueries.add(pqF("c", "d", "e"));+ assertCollectionQueries(+ fq.expand(flatQueries),+ pqF("a", "b", "c"),+ pqF("c", "d", "e"),+ pqF("a", "b", "c", "d", "e"));// "a b c d","b c" => "a b c d","b c"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "c", "d" ) );- flatQueries.add( pqF( "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "c", "d" ), pqF( "b", "c" ) );+ flatQueries.add(pqF("a", "b", "c", "d"));+ flatQueries.add(pqF("b", "c"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b", "c", "d"), pqF("b", "c"));// "a b b","b c" => "a b b","b c","a b b c"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "b" ) );- flatQueries.add( pqF( "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "b" ), pqF( "b", "c" ), pqF( "a", "b", "b", "c" ) );+ flatQueries.add(pqF("a", "b", "b"));+ flatQueries.add(pqF("b", "c"));+ assertCollectionQueries(+ fq.expand(flatQueries), pqF("a", "b", "b"), pqF("b", "c"), pqF("a", "b", "b", "c"));// "a b","b a" => "a b","b a","a b a", "b a b"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "b", "a" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "b", "a" ), pqF( "a", "b", "a" ), pqF( "b", "a", "b" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("b", "a"));+ assertCollectionQueries(+ fq.expand(flatQueries),+ pqF("a", "b"),+ pqF("b", "a"),+ pqF("a", "b", "a"),+ pqF("b", "a", "b"));// "a b","a b c" => "a b","a b c"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "a", "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "a", "b", "c" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("a", "b", "c"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b"), pqF("a", "b", "c"));}public void testNoExpand() throws Exception {- Query dummy = pqF( "DUMMY" );- FieldQuery fq = new FieldQuery( dummy, true, true );+ Query dummy = pqF("DUMMY");+ FieldQuery fq = new FieldQuery(dummy, true, true);// "a b","c d" => "a b","c d"Set<Query> flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "c", "d" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "c", "d" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("c", "d"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b"), pqF("c", "d"));// "a","a b" => "a", "a b"flatQueries = new HashSet<>();- flatQueries.add( tq( "a" ) );- flatQueries.add( pqF( "a", "b" ) );- assertCollectionQueries( fq.expand( flatQueries ),- tq( "a" ), pqF( "a", "b" ) );+ flatQueries.add(tq("a"));+ flatQueries.add(pqF("a", "b"));+ assertCollectionQueries(fq.expand(flatQueries), tq("a"), pqF("a", "b"));// "a b","b" => "a b", "b"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( tq( "b" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), tq( "b" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(tq("b"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b"), tq("b"));// "a b c","b c" => "a b c","b c"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "c" ) );- flatQueries.add( pqF( "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "c" ), pqF( "b", "c" ) );+ flatQueries.add(pqF("a", "b", "c"));+ flatQueries.add(pqF("b", "c"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b", "c"), pqF("b", "c"));// "a b","a b c" => "a b","a b c"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b" ) );- flatQueries.add( pqF( "a", "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b" ), pqF( "a", "b", "c" ) );+ flatQueries.add(pqF("a", "b"));+ flatQueries.add(pqF("a", "b", "c"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b"), pqF("a", "b", "c"));// "a b c","b d e" => "a b c","b d e"flatQueries = new HashSet<>();- flatQueries.add( pqF( "a", "b", "c" ) );- flatQueries.add( pqF( "b", "d", "e" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pqF( "a", "b", "c" ), pqF( "b", "d", "e" ) );+ flatQueries.add(pqF("a", "b", "c"));+ flatQueries.add(pqF("b", "d", "e"));+ assertCollectionQueries(fq.expand(flatQueries), pqF("a", "b", "c"), pqF("b", "d", "e"));}public void testExpandNotFieldMatch() throws Exception {- Query dummy = pqF( "DUMMY" );- FieldQuery fq = new FieldQuery( dummy, true, false );+ Query dummy = pqF("DUMMY");+ FieldQuery fq = new FieldQuery(dummy, true, false);// f1:"a b",f2:"b c" => f1:"a b",f2:"b c",f1:"a b c"Set<Query> flatQueries = new HashSet<>();- flatQueries.add( pq( F1, "a", "b" ) );- flatQueries.add( pq( F2, "b", "c" ) );- assertCollectionQueries( fq.expand( flatQueries ),- pq( F1, "a", "b" ), pq( F2, "b", "c" ), pq( F1, "a", "b", "c" ) );+ flatQueries.add(pq(F1, "a", "b"));+ flatQueries.add(pq(F2, "b", "c"));+ assertCollectionQueries(+ fq.expand(flatQueries), pq(F1, "a", "b"), pq(F2, "b", "c"), pq(F1, "a", "b", "c"));}public void testGetFieldTermMap() throws Exception {- Query query = tq( "a" );- FieldQuery fq = new FieldQuery( query, true, true );+ Query query = tq("a");+ FieldQuery fq = new FieldQuery(query, true, true);- QueryPhraseMap pqm = fq.getFieldTermMap( F, "a" );- assertNotNull( pqm );- assertTrue( pqm.isTerminal() );+ QueryPhraseMap pqm = fq.getFieldTermMap(F, "a");+ assertNotNull(pqm);+ assertTrue(pqm.isTerminal());- pqm = fq.getFieldTermMap( F, "b" );- assertNull( pqm );+ pqm = fq.getFieldTermMap(F, "b");+ assertNull(pqm);- pqm = fq.getFieldTermMap( F1, "a" );- assertNull( pqm );+ pqm = fq.getFieldTermMap(F1, "a");+ assertNull(pqm);}public void testGetRootMap() throws Exception {- Query dummy = pqF( "DUMMY" );- FieldQuery fq = new FieldQuery( dummy, true, true );-- QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) );- QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) );- assertTrue( rootMap1 == rootMap2 );- QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) );- assertTrue( rootMap1 == rootMap3 );- QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) );- assertFalse( rootMap4 == rootMap3 );+ Query dummy = pqF("DUMMY");+ FieldQuery fq = new FieldQuery(dummy, true, true);++ QueryPhraseMap rootMap1 = fq.getRootMap(tq("a"));+ QueryPhraseMap rootMap2 = fq.getRootMap(tq("a"));+ assertTrue(rootMap1 == rootMap2);+ QueryPhraseMap rootMap3 = fq.getRootMap(tq("b"));+ assertTrue(rootMap1 == rootMap3);+ QueryPhraseMap rootMap4 = fq.getRootMap(tq(F1, "b"));+ assertFalse(rootMap4 == rootMap3);}public void testGetRootMapNotFieldMatch() throws Exception {- Query dummy = pqF( "DUMMY" );- FieldQuery fq = new FieldQuery( dummy, true, false );-- QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) );- QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) );- assertTrue( rootMap1 == rootMap2 );- QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) );- assertTrue( rootMap1 == rootMap3 );- QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) );- assertTrue( rootMap4 == rootMap3 );+ Query dummy = pqF("DUMMY");+ FieldQuery fq = new FieldQuery(dummy, true, false);++ QueryPhraseMap rootMap1 = fq.getRootMap(tq("a"));+ QueryPhraseMap rootMap2 = fq.getRootMap(tq("a"));+ assertTrue(rootMap1 == rootMap2);+ QueryPhraseMap rootMap3 = fq.getRootMap(tq("b"));+ assertTrue(rootMap1 == rootMap3);+ QueryPhraseMap rootMap4 = fq.getRootMap(tq(F1, "b"));+ assertTrue(rootMap4 == rootMap3);}public void testGetTermSet() throws Exception {@@ -286,297 +282,297 @@ public class TestFieldQuery extends AbstractTestCase {innerQuery.add(new TermQuery(new Term(F, "E")), Occur.MUST);query.add(innerQuery.build(), Occur.MUST_NOT);- FieldQuery fq = new FieldQuery( query.build(), true, true );- assertEquals( 2, fq.termSetMap.size() );- Set<String> termSet = fq.getTermSet( F );- assertEquals( 2, termSet.size() );- assertTrue( termSet.contains( "A" ) );- assertTrue( termSet.contains( "B" ) );- termSet = fq.getTermSet( "x" );- assertEquals( 1, termSet.size() );- assertTrue( termSet.contains( "C" ) );- termSet = fq.getTermSet( "y" );- assertNull( termSet );+ FieldQuery fq = new FieldQuery(query.build(), true, true);+ assertEquals(2, fq.termSetMap.size());+ Set<String> termSet = fq.getTermSet(F);+ assertEquals(2, termSet.size());+ assertTrue(termSet.contains("A"));+ assertTrue(termSet.contains("B"));+ termSet = fq.getTermSet("x");+ assertEquals(1, termSet.size());+ assertTrue(termSet.contains("C"));+ termSet = fq.getTermSet("y");+ assertNull(termSet);}public void testQueryPhraseMap1Term() throws Exception {- Query query = tq( "a" );+ Query query = tq("a");// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );- assertTrue( qpm.subMap.get( "a" ) != null );- assertTrue( qpm.subMap.get( "a" ).terminal );- assertEquals( 1F, qpm.subMap.get( "a" ).boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());+ assertTrue(qpm.subMap.get("a") != null);+ assertTrue(qpm.subMap.get("a").terminal);+ assertEquals(1F, qpm.subMap.get("a").boost, 0);// phraseHighlight = true, fieldMatch = false- fq = new FieldQuery( query, true, false );+ fq = new FieldQuery(query, true, false);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( F ) );- assertNotNull( map.get( null ) );- qpm = map.get( null );- assertEquals( 1, qpm.subMap.size() );- assertTrue( qpm.subMap.get( "a" ) != null );- assertTrue( qpm.subMap.get( "a" ).terminal );- assertEquals( 1F, qpm.subMap.get( "a" ).boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(F));+ assertNotNull(map.get(null));+ qpm = map.get(null);+ assertEquals(1, qpm.subMap.size());+ assertTrue(qpm.subMap.get("a") != null);+ assertTrue(qpm.subMap.get("a").terminal);+ assertEquals(1F, qpm.subMap.get("a").boost, 0);// phraseHighlight = false, fieldMatch = true- fq = new FieldQuery( query, false, true );+ fq = new FieldQuery(query, false, true);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );- assertTrue( qpm.subMap.get( "a" ) != null );- assertTrue( qpm.subMap.get( "a" ).terminal );- assertEquals( 1F, qpm.subMap.get( "a" ).boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());+ assertTrue(qpm.subMap.get("a") != null);+ assertTrue(qpm.subMap.get("a").terminal);+ assertEquals(1F, qpm.subMap.get("a").boost, 0);// phraseHighlight = false, fieldMatch = false- fq = new FieldQuery( query, false, false );+ fq = new FieldQuery(query, false, false);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( F ) );- assertNotNull( map.get( null ) );- qpm = map.get( null );- assertEquals( 1, qpm.subMap.size() );- assertTrue( qpm.subMap.get( "a" ) != null );- assertTrue( qpm.subMap.get( "a" ).terminal );- assertEquals( 1F, qpm.subMap.get( "a" ).boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(F));+ assertNotNull(map.get(null));+ qpm = map.get(null);+ assertEquals(1, qpm.subMap.size());+ assertTrue(qpm.subMap.get("a") != null);+ assertTrue(qpm.subMap.get("a").terminal);+ assertEquals(1F, qpm.subMap.get("a").boost, 0);// boost != 1- query = tq( 2, "a" );- fq = new FieldQuery( query, true, true );+ query = tq(2, "a");+ fq = new FieldQuery(query, true, true);map = fq.rootMaps;- qpm = map.get( F );- assertEquals( 2F, qpm.subMap.get( "a" ).boost, 0);+ qpm = map.get(F);+ assertEquals(2F, qpm.subMap.get("a").boost, 0);}public void testQueryPhraseMap1Phrase() throws Exception {- Query query = pqF( "a", "b" );+ Query query = pqF("a", "b");// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// phraseHighlight = true, fieldMatch = false- fq = new FieldQuery( query, true, false );+ fq = new FieldQuery(query, true, false);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( F ) );- assertNotNull( map.get( null ) );- qpm = map.get( null );- assertEquals( 1, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "a" ) );- qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(F));+ assertNotNull(map.get(null));+ qpm = map.get(null);+ assertEquals(1, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("a"));+ qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// phraseHighlight = false, fieldMatch = true- fq = new FieldQuery( query, false, true );+ fq = new FieldQuery(query, false, true);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- qpm = map.get( F );- assertEquals( 2, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "a" ) );- qpm2 = qpm.subMap.get( "a" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);-- assertNotNull( qpm.subMap.get( "b" ) );- qpm2 = qpm.subMap.get( "b" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ qpm = map.get(F);+ assertEquals(2, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("a"));+ qpm2 = qpm.subMap.get("a");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);++ assertNotNull(qpm.subMap.get("b"));+ qpm2 = qpm.subMap.get("b");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);// phraseHighlight = false, fieldMatch = false- fq = new FieldQuery( query, false, false );+ fq = new FieldQuery(query, false, false);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( F ) );- assertNotNull( map.get( null ) );- qpm = map.get( null );- assertEquals( 2, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "a" ) );- qpm2 = qpm.subMap.get( "a" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);-- assertNotNull( qpm.subMap.get( "b" ) );- qpm2 = qpm.subMap.get( "b" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(F));+ assertNotNull(map.get(null));+ qpm = map.get(null);+ assertEquals(2, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("a"));+ qpm2 = qpm.subMap.get("a");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);++ assertNotNull(qpm.subMap.get("b"));+ qpm2 = qpm.subMap.get("b");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);// boost != 1- query = pqF( 2, "a", "b" );+ query = pqF(2, "a", "b");// phraseHighlight = false, fieldMatch = false- fq = new FieldQuery( query, false, false );+ fq = new FieldQuery(query, false, false);map = fq.rootMaps;- qpm = map.get( null );- qpm2 = qpm.subMap.get( "a" );- assertEquals( 2F, qpm2.boost, 0);- qpm3 = qpm2.subMap.get( "b" );- assertEquals( 2F, qpm3.boost, 0);- qpm2 = qpm.subMap.get( "b" );- assertEquals( 2F, qpm2.boost, 0);+ qpm = map.get(null);+ qpm2 = qpm.subMap.get("a");+ assertEquals(2F, qpm2.boost, 0);+ qpm3 = qpm2.subMap.get("b");+ assertEquals(2F, qpm3.boost, 0);+ qpm2 = qpm.subMap.get("b");+ assertEquals(2F, qpm2.boost, 0);}public void testQueryPhraseMap1PhraseAnother() throws Exception {- Query query = pqF( "search", "engines" );+ Query query = pqF("search", "engines");// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "search" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "search" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "engines" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "engines" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("search"));+ QueryPhraseMap qpm2 = qpm.subMap.get("search");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("engines"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("engines");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);}public void testQueryPhraseMap2Phrases() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "b" ), Occur.SHOULD );- query.add( pqF( 2, "c", "d" ), Occur.SHOULD );+ query.add(pqF("a", "b"), Occur.SHOULD);+ query.add(pqF(2, "c", "d"), Occur.SHOULD);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 2, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(2, qpm.subMap.size());// "a b"- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "c d"^2- assertNotNull( qpm.subMap.get( "c" ) );- qpm2 = qpm.subMap.get( "c" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "d" ) );- qpm3 = qpm2.subMap.get( "d" );- assertTrue( qpm3.terminal );- assertEquals( 2F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("c"));+ qpm2 = qpm.subMap.get("c");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("d"));+ qpm3 = qpm2.subMap.get("d");+ assertTrue(qpm3.terminal);+ assertEquals(2F, qpm3.boost, 0);}public void testQueryPhraseMap2PhrasesFields() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pq( F1, "a", "b" ), Occur.SHOULD );- query.add( pq( 2F, F2, "c", "d" ), Occur.SHOULD );+ query.add(pq(F1, "a", "b"), Occur.SHOULD);+ query.add(pq(2F, F2, "c", "d"), Occur.SHOULD);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 2, map.size() );- assertNull( map.get( null ) );+ assertEquals(2, map.size());+ assertNull(map.get(null));// "a b"- assertNotNull( map.get( F1 ) );- QueryPhraseMap qpm = map.get( F1 );- assertEquals( 1, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(map.get(F1));+ QueryPhraseMap qpm = map.get(F1);+ assertEquals(1, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "c d"^2- assertNotNull( map.get( F2 ) );- qpm = map.get( F2 );- assertEquals( 1, qpm.subMap.size() );- assertNotNull( qpm.subMap.get( "c" ) );- qpm2 = qpm.subMap.get( "c" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "d" ) );- qpm3 = qpm2.subMap.get( "d" );- assertTrue( qpm3.terminal );- assertEquals( 2F, qpm3.boost, 0);+ assertNotNull(map.get(F2));+ qpm = map.get(F2);+ assertEquals(1, qpm.subMap.size());+ assertNotNull(qpm.subMap.get("c"));+ qpm2 = qpm.subMap.get("c");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("d"));+ qpm3 = qpm2.subMap.get("d");+ assertTrue(qpm3.terminal);+ assertEquals(2F, qpm3.boost, 0);// phraseHighlight = true, fieldMatch = false- fq = new FieldQuery( query.build(), true, false );+ fq = new FieldQuery(query.build(), true, false);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( F1 ) );- assertNull( map.get( F2 ) );- assertNotNull( map.get( null ) );- qpm = map.get( null );- assertEquals( 2, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(F1));+ assertNull(map.get(F2));+ assertNotNull(map.get(null));+ qpm = map.get(null);+ assertEquals(2, qpm.subMap.size());// "a b"- assertNotNull( qpm.subMap.get( "a" ) );- qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("a"));+ qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "c d"^2- assertNotNull( qpm.subMap.get( "c" ) );- qpm2 = qpm.subMap.get( "c" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "d" ) );- qpm3 = qpm2.subMap.get( "d" );- assertTrue( qpm3.terminal );- assertEquals( 2F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("c"));+ qpm2 = qpm.subMap.get("c");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("d"));+ qpm3 = qpm2.subMap.get("d");+ assertTrue(qpm3.terminal);+ assertEquals(2F, qpm3.boost, 0);}/*@@ -589,54 +585,54 @@ public class TestFieldQuery extends AbstractTestCase {*/public void testQueryPhraseMapOverlapPhrases() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "b", "c" ), Occur.SHOULD );- query.add( pqF( 2, "b", "c", "d" ), Occur.SHOULD );- query.add( pqF( 3, "b", "d" ), Occur.SHOULD );+ query.add(pqF("a", "b", "c"), Occur.SHOULD);+ query.add(pqF(2, "b", "c", "d"), Occur.SHOULD);+ query.add(pqF(3, "b", "d"), Occur.SHOULD);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 2, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(2, qpm.subMap.size());// "a b c"- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );- assertFalse( qpm3.terminal );- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "c" ) );- QueryPhraseMap qpm4 = qpm3.subMap.get( "c" );- assertTrue( qpm4.terminal );- assertEquals( 1F, qpm4.boost, 0);- assertNotNull( qpm4.subMap.get( "d" ) );- QueryPhraseMap qpm5 = qpm4.subMap.get( "d" );- assertTrue( qpm5.terminal );- assertEquals( 1F, qpm5.boost, 0);+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("b");+ assertFalse(qpm3.terminal);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("c"));+ QueryPhraseMap qpm4 = qpm3.subMap.get("c");+ assertTrue(qpm4.terminal);+ assertEquals(1F, qpm4.boost, 0);+ assertNotNull(qpm4.subMap.get("d"));+ QueryPhraseMap qpm5 = qpm4.subMap.get("d");+ assertTrue(qpm5.terminal);+ assertEquals(1F, qpm5.boost, 0);// "b c d"^2, "b d"^3- assertNotNull( qpm.subMap.get( "b" ) );- qpm2 = qpm.subMap.get( "b" );- assertFalse( qpm2.terminal );- assertEquals( 2, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "c" ) );- qpm3 = qpm2.subMap.get( "c" );- assertFalse( qpm3.terminal );- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "d" ) );- qpm4 = qpm3.subMap.get( "d" );- assertTrue( qpm4.terminal );- assertEquals( 2F, qpm4.boost, 0);- assertNotNull( qpm2.subMap.get( "d" ) );- qpm3 = qpm2.subMap.get( "d" );- assertTrue( qpm3.terminal );- assertEquals( 3F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("b"));+ qpm2 = qpm.subMap.get("b");+ assertFalse(qpm2.terminal);+ assertEquals(2, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("c"));+ qpm3 = qpm2.subMap.get("c");+ assertFalse(qpm3.terminal);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("d"));+ qpm4 = qpm3.subMap.get("d");+ assertTrue(qpm4.terminal);+ assertEquals(2F, qpm4.boost, 0);+ assertNotNull(qpm2.subMap.get("d"));+ qpm3 = qpm2.subMap.get("d");+ assertTrue(qpm3.terminal);+ assertEquals(3F, qpm3.boost, 0);}/*@@ -647,34 +643,34 @@ public class TestFieldQuery extends AbstractTestCase {*/public void testQueryPhraseMapOverlapPhrases2() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "b" ), Occur.SHOULD );- query.add( pqF( 2, "a", "b", "c" ), Occur.SHOULD );+ query.add(pqF("a", "b"), Occur.SHOULD);+ query.add(pqF(2, "a", "b", "c"), Occur.SHOULD);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());// "a b"- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "b" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "b" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("b"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("b");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "a b c"^2- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "c" ) );- QueryPhraseMap qpm4 = qpm3.subMap.get( "c" );- assertTrue( qpm4.terminal );- assertEquals( 2F, qpm4.boost, 0);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("c"));+ QueryPhraseMap qpm4 = qpm3.subMap.get("c");+ assertTrue(qpm4.terminal);+ assertEquals(2F, qpm4.boost, 0);}/*@@ -687,48 +683,48 @@ public class TestFieldQuery extends AbstractTestCase {*/public void testQueryPhraseMapOverlapPhrases3() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "a", "a", "a", "a" ), Occur.SHOULD );- query.add( pqF( 2, "a", "a", "a" ), Occur.SHOULD );+ query.add(pqF("a", "a", "a", "a"), Occur.SHOULD);+ query.add(pqF(2, "a", "a", "a"), Occur.SHOULD);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 1, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(1, qpm.subMap.size());// "a a a"- assertNotNull( qpm.subMap.get( "a" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "a" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "a" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "a" );- assertFalse( qpm3.terminal );- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "a" ) );- QueryPhraseMap qpm4 = qpm3.subMap.get( "a" );- assertTrue( qpm4.terminal );+ assertNotNull(qpm.subMap.get("a"));+ QueryPhraseMap qpm2 = qpm.subMap.get("a");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("a"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("a");+ assertFalse(qpm3.terminal);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("a"));+ QueryPhraseMap qpm4 = qpm3.subMap.get("a");+ assertTrue(qpm4.terminal);// "a a a a"- assertEquals( 1, qpm4.subMap.size() );- assertNotNull( qpm4.subMap.get( "a" ) );- QueryPhraseMap qpm5 = qpm4.subMap.get( "a" );- assertTrue( qpm5.terminal );+ assertEquals(1, qpm4.subMap.size());+ assertNotNull(qpm4.subMap.get("a"));+ QueryPhraseMap qpm5 = qpm4.subMap.get("a");+ assertTrue(qpm5.terminal);// "a a a a a"- assertEquals( 1, qpm5.subMap.size() );- assertNotNull( qpm5.subMap.get( "a" ) );- QueryPhraseMap qpm6 = qpm5.subMap.get( "a" );- assertTrue( qpm6.terminal );+ assertEquals(1, qpm5.subMap.size());+ assertNotNull(qpm5.subMap.get("a"));+ QueryPhraseMap qpm6 = qpm5.subMap.get("a");+ assertTrue(qpm6.terminal);// "a a a a a a"- assertEquals( 1, qpm6.subMap.size() );- assertNotNull( qpm6.subMap.get( "a" ) );- QueryPhraseMap qpm7 = qpm6.subMap.get( "a" );- assertTrue( qpm7.terminal );+ assertEquals(1, qpm6.subMap.size());+ assertNotNull(qpm6.subMap.get("a"));+ QueryPhraseMap qpm7 = qpm6.subMap.get("a");+ assertTrue(qpm7.terminal);}public void testQueryPhraseMapOverlap2gram() throws Exception {@@ -737,157 +733,157 @@ public class TestFieldQuery extends AbstractTestCase {query.add(toPhraseQuery(analyze("bcd", F, analyzerB), F), Occur.MUST);// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query.build(), true, true );+ FieldQuery fq = new FieldQuery(query.build(), true, true);Map<String, QueryPhraseMap> map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- QueryPhraseMap qpm = map.get( F );- assertEquals( 2, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ QueryPhraseMap qpm = map.get(F);+ assertEquals(2, qpm.subMap.size());// "ab bc"- assertNotNull( qpm.subMap.get( "ab" ) );- QueryPhraseMap qpm2 = qpm.subMap.get( "ab" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "bc" ) );- QueryPhraseMap qpm3 = qpm2.subMap.get( "bc" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("ab"));+ QueryPhraseMap qpm2 = qpm.subMap.get("ab");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("bc"));+ QueryPhraseMap qpm3 = qpm2.subMap.get("bc");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "ab bc cd"- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "cd" ) );- QueryPhraseMap qpm4 = qpm3.subMap.get( "cd" );- assertTrue( qpm4.terminal );- assertEquals( 1F, qpm4.boost, 0);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("cd"));+ QueryPhraseMap qpm4 = qpm3.subMap.get("cd");+ assertTrue(qpm4.terminal);+ assertEquals(1F, qpm4.boost, 0);// "bc cd"- assertNotNull( qpm.subMap.get( "bc" ) );- qpm2 = qpm.subMap.get( "bc" );- assertFalse( qpm2.terminal );- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "cd" ) );- qpm3 = qpm2.subMap.get( "cd" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("bc"));+ qpm2 = qpm.subMap.get("bc");+ assertFalse(qpm2.terminal);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("cd"));+ qpm3 = qpm2.subMap.get("cd");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// phraseHighlight = false, fieldMatch = true- fq = new FieldQuery( query.build(), false, true );+ fq = new FieldQuery(query.build(), false, true);map = fq.rootMaps;- assertEquals( 1, map.size() );- assertNull( map.get( null ) );- assertNotNull( map.get( F ) );- qpm = map.get( F );- assertEquals( 3, qpm.subMap.size() );+ assertEquals(1, map.size());+ assertNull(map.get(null));+ assertNotNull(map.get(F));+ qpm = map.get(F);+ assertEquals(3, qpm.subMap.size());// "ab bc"- assertNotNull( qpm.subMap.get( "ab" ) );- qpm2 = qpm.subMap.get( "ab" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "bc" ) );- qpm3 = qpm2.subMap.get( "bc" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("ab"));+ qpm2 = qpm.subMap.get("ab");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("bc"));+ qpm3 = qpm2.subMap.get("bc");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "ab bc cd"- assertEquals( 1, qpm3.subMap.size() );- assertNotNull( qpm3.subMap.get( "cd" ) );- qpm4 = qpm3.subMap.get( "cd" );- assertTrue( qpm4.terminal );- assertEquals( 1F, qpm4.boost, 0);+ assertEquals(1, qpm3.subMap.size());+ assertNotNull(qpm3.subMap.get("cd"));+ qpm4 = qpm3.subMap.get("cd");+ assertTrue(qpm4.terminal);+ assertEquals(1F, qpm4.boost, 0);// "bc cd"- assertNotNull( qpm.subMap.get( "bc" ) );- qpm2 = qpm.subMap.get( "bc" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);- assertEquals( 1, qpm2.subMap.size() );- assertNotNull( qpm2.subMap.get( "cd" ) );- qpm3 = qpm2.subMap.get( "cd" );- assertTrue( qpm3.terminal );- assertEquals( 1F, qpm3.boost, 0);+ assertNotNull(qpm.subMap.get("bc"));+ qpm2 = qpm.subMap.get("bc");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);+ assertEquals(1, qpm2.subMap.size());+ assertNotNull(qpm2.subMap.get("cd"));+ qpm3 = qpm2.subMap.get("cd");+ assertTrue(qpm3.terminal);+ assertEquals(1F, qpm3.boost, 0);// "cd"- assertNotNull( qpm.subMap.get( "cd" ) );- qpm2 = qpm.subMap.get( "cd" );- assertTrue( qpm2.terminal );- assertEquals( 1F, qpm2.boost, 0);- assertEquals( 0, qpm2.subMap.size() );+ assertNotNull(qpm.subMap.get("cd"));+ qpm2 = qpm.subMap.get("cd");+ assertTrue(qpm2.terminal);+ assertEquals(1F, qpm2.boost, 0);+ assertEquals(0, qpm2.subMap.size());}public void testSearchPhrase() throws Exception {- Query query = pqF( "a", "b", "c" );+ Query query = pqF("a", "b", "c");// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);// "a"List<TermInfo> phraseCandidate = new ArrayList<>();- phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) );- assertNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("a", 0, 1, 0, 1));+ assertNull(fq.searchPhrase(F, phraseCandidate));// "a b"- phraseCandidate.add( new TermInfo( "b", 2, 3, 1, 1 ) );- assertNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("b", 2, 3, 1, 1));+ assertNull(fq.searchPhrase(F, phraseCandidate));// "a b c"- phraseCandidate.add( new TermInfo( "c", 4, 5, 2, 1 ) );- assertNotNull( fq.searchPhrase( F, phraseCandidate ) );- assertNull( fq.searchPhrase( "x", phraseCandidate ) );+ phraseCandidate.add(new TermInfo("c", 4, 5, 2, 1));+ assertNotNull(fq.searchPhrase(F, phraseCandidate));+ assertNull(fq.searchPhrase("x", phraseCandidate));// phraseHighlight = true, fieldMatch = false- fq = new FieldQuery( query, true, false );+ fq = new FieldQuery(query, true, false);// "a b c"- assertNotNull( fq.searchPhrase( F, phraseCandidate ) );- assertNotNull( fq.searchPhrase( "x", phraseCandidate ) );+ assertNotNull(fq.searchPhrase(F, phraseCandidate));+ assertNotNull(fq.searchPhrase("x", phraseCandidate));// phraseHighlight = false, fieldMatch = true- fq = new FieldQuery( query, false, true );+ fq = new FieldQuery(query, false, true);// "a"phraseCandidate.clear();- phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) );- assertNotNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("a", 0, 1, 0, 1));+ assertNotNull(fq.searchPhrase(F, phraseCandidate));// "a b"- phraseCandidate.add( new TermInfo( "b", 2, 3, 1, 1 ) );- assertNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("b", 2, 3, 1, 1));+ assertNull(fq.searchPhrase(F, phraseCandidate));// "a b c"- phraseCandidate.add( new TermInfo( "c", 4, 5, 2, 1 ) );- assertNotNull( fq.searchPhrase( F, phraseCandidate ) );- assertNull( fq.searchPhrase( "x", phraseCandidate ) );+ phraseCandidate.add(new TermInfo("c", 4, 5, 2, 1));+ assertNotNull(fq.searchPhrase(F, phraseCandidate));+ assertNull(fq.searchPhrase("x", phraseCandidate));}public void testSearchPhraseSlop() throws Exception {// "a b c"~0- Query query = pqF( "a", "b", "c" );+ Query query = pqF("a", "b", "c");// phraseHighlight = true, fieldMatch = true- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);// "a b c" w/ position-gap = 2List<TermInfo> phraseCandidate = new ArrayList<>();- phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) );- phraseCandidate.add( new TermInfo( "b", 2, 3, 2, 1 ) );- phraseCandidate.add( new TermInfo( "c", 4, 5, 4, 1 ) );- assertNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("a", 0, 1, 0, 1));+ phraseCandidate.add(new TermInfo("b", 2, 3, 2, 1));+ phraseCandidate.add(new TermInfo("c", 4, 5, 4, 1));+ assertNull(fq.searchPhrase(F, phraseCandidate));// "a b c"~1- query = pqF( 1F, 1, "a", "b", "c" );+ query = pqF(1F, 1, "a", "b", "c");// phraseHighlight = true, fieldMatch = true- fq = new FieldQuery( query, true, true );+ fq = new FieldQuery(query, true, true);// "a b c" w/ position-gap = 2- assertNotNull( fq.searchPhrase( F, phraseCandidate ) );+ assertNotNull(fq.searchPhrase(F, phraseCandidate));// "a b c" w/ position-gap = 3phraseCandidate.clear();- phraseCandidate.add( new TermInfo( "a", 0, 1, 0, 1 ) );- phraseCandidate.add( new TermInfo( "b", 2, 3, 3, 1 ) );- phraseCandidate.add( new TermInfo( "c", 4, 5, 6, 1 ) );- assertNull( fq.searchPhrase( F, phraseCandidate ) );+ phraseCandidate.add(new TermInfo("a", 0, 1, 0, 1));+ phraseCandidate.add(new TermInfo("b", 2, 3, 3, 1));+ phraseCandidate.add(new TermInfo("c", 4, 5, 6, 1));+ assertNull(fq.searchPhrase(F, phraseCandidate));}public void testHighlightQuery() throws Exception {@@ -908,54 +904,53 @@ public class TestFieldQuery extends AbstractTestCase {public void testRangeQuery() throws Exception {makeIndexStrMV();- defgMultiTermQueryTest(new TermRangeQuery (F, new BytesRef("d"), new BytesRef("e"), true, true));+ defgMultiTermQueryTest(new TermRangeQuery(F, new BytesRef("d"), new BytesRef("e"), true, true));}private void defgMultiTermQueryTest(Query query) throws IOException {- FieldQuery fq = new FieldQuery( query, reader, true, true );+ FieldQuery fq = new FieldQuery(query, reader, true, true);QueryPhraseMap qpm = fq.getFieldTermMap(F, "defg");- assertNotNull (qpm);- assertNull (fq.getFieldTermMap(F, "dog"));+ assertNotNull(qpm);+ assertNull(fq.getFieldTermMap(F, "dog"));List<TermInfo> phraseCandidate = new ArrayList<>();- phraseCandidate.add( new TermInfo( "defg", 0, 12, 0, 1 ) );- assertNotNull (fq.searchPhrase(F, phraseCandidate));+ phraseCandidate.add(new TermInfo("defg", 0, 12, 0, 1));+ assertNotNull(fq.searchPhrase(F, phraseCandidate));}public void testStopRewrite() throws Exception {- Query q = new Query() {-- @Override- public String toString(String field) {- return "DummyQuery";- }-- @Override- public void visit(QueryVisitor visitor) {-- }-- @Override- public boolean equals(Object o) {- throw new AssertionError();- }-- @Override- public int hashCode() {- throw new AssertionError();- }- };- make1d1fIndex( "a" );+ Query q =+ new Query() {++ @Override+ public String toString(String field) {+ return "DummyQuery";+ }++ @Override+ public void visit(QueryVisitor visitor) {}++ @Override+ public boolean equals(Object o) {+ throw new AssertionError();+ }++ @Override+ public int hashCode() {+ throw new AssertionError();+ }+ };+ make1d1fIndex("a");assertNotNull(reader);- new FieldQuery(q, reader, true, true );+ new FieldQuery(q, reader, true, true);}public void testFlattenConstantScoreQuery() throws Exception {initBoost();- Query query = new ConstantScoreQuery(pqF( "A" ));+ Query query = new ConstantScoreQuery(pqF("A"));query = new BoostQuery(query, boost);- FieldQuery fq = new FieldQuery( query, true, true );+ FieldQuery fq = new FieldQuery(query, true, true);Set<Query> flatQueries = new HashSet<>();- fq.flatten( query, reader, flatQueries, 1f );- assertCollectionQueries( flatQueries, tq( boost, "A" ) );+ fq.flatten(query, reader, flatQueries, 1f);+ assertCollectionQueries(flatQueries, tq(boost, "A"));}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldTermStack.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldTermStack.javaindex 5284ba9cb1a..72efd59f99b 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldTermStack.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldTermStack.java@@ -15,59 +15,60 @@* limitations under the License.*/package org.apache.lucene.search.vectorhighlight;+import org.apache.lucene.index.Term;-import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.WildcardQuery;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;import org.apache.lucene.util.TestUtil;public class TestFieldTermStack extends AbstractTestCase {-+public void test1Term() throws Exception {makeIndex();-- FieldQuery fq = new FieldQuery( tq( "a" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 6, stack.termList.size() );- assertEquals( "a(0,1,0)", stack.pop().toString() );- assertEquals( "a(2,3,1)", stack.pop().toString() );- assertEquals( "a(4,5,2)", stack.pop().toString() );- assertEquals( "a(12,13,6)", stack.pop().toString() );- assertEquals( "a(28,29,14)", stack.pop().toString() );- assertEquals( "a(32,33,16)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("a"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(6, stack.termList.size());+ assertEquals("a(0,1,0)", stack.pop().toString());+ assertEquals("a(2,3,1)", stack.pop().toString());+ assertEquals("a(4,5,2)", stack.pop().toString());+ assertEquals("a(12,13,6)", stack.pop().toString());+ assertEquals("a(28,29,14)", stack.pop().toString());+ assertEquals("a(32,33,16)", stack.pop().toString());}-+public void test2Terms() throws Exception {makeIndex();-+BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( tq( "b" ), Occur.SHOULD );- query.add( tq( "c" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( query.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 8, stack.termList.size() );- assertEquals( "b(6,7,3)", stack.pop().toString() );- assertEquals( "b(8,9,4)", stack.pop().toString() );- assertEquals( "c(10,11,5)", stack.pop().toString() );- assertEquals( "b(14,15,7)", stack.pop().toString() );- assertEquals( "b(16,17,8)", stack.pop().toString() );- assertEquals( "c(18,19,9)", stack.pop().toString() );- assertEquals( "b(26,27,13)", stack.pop().toString() );- assertEquals( "b(30,31,15)", stack.pop().toString() );+ query.add(tq("b"), Occur.SHOULD);+ query.add(tq("c"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(query.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(8, stack.termList.size());+ assertEquals("b(6,7,3)", stack.pop().toString());+ assertEquals("b(8,9,4)", stack.pop().toString());+ assertEquals("c(10,11,5)", stack.pop().toString());+ assertEquals("b(14,15,7)", stack.pop().toString());+ assertEquals("b(16,17,8)", stack.pop().toString());+ assertEquals("c(18,19,9)", stack.pop().toString());+ assertEquals("b(26,27,13)", stack.pop().toString());+ assertEquals("b(30,31,15)", stack.pop().toString());}-+public void test1Phrase() throws Exception {makeIndex();-- FieldQuery fq = new FieldQuery( pqF( "c", "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 3, stack.termList.size() );- assertEquals( "c(10,11,5)", stack.pop().toString() );- assertEquals( "c(18,19,9)", stack.pop().toString() );- assertEquals( "d(20,21,10)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(pqF("c", "d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(3, stack.termList.size());+ assertEquals("c(10,11,5)", stack.pop().toString());+ assertEquals("c(18,19,9)", stack.pop().toString());+ assertEquals("d(20,21,10)", stack.pop().toString());}-+private void makeIndex() throws Exception {// 111111111122222// 0123456789012345678901234 (offsets)@@ -77,134 +78,135 @@ public class TestFieldTermStack extends AbstractTestCase {// 222233333// 678901234 (offsets)// b a b a f- //1314151617 (position)+ // 1314151617 (position)String value2 = "b a b a f";-- make1dmfIndex( value1, value2 );++ make1dmfIndex(value1, value2);}-+public void test1TermB() throws Exception {makeIndexB();-- FieldQuery fq = new FieldQuery( tq( "ab" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 2, stack.termList.size() );- assertEquals( "ab(2,4,2)", stack.pop().toString() );- assertEquals( "ab(6,8,6)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("ab"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(2, stack.termList.size());+ assertEquals("ab(2,4,2)", stack.pop().toString());+ assertEquals("ab(6,8,6)", stack.pop().toString());}-+public void test2TermsB() throws Exception {makeIndexB();-+BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( tq( "bc" ), Occur.SHOULD );- query.add( tq( "ef" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( query.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 3, stack.termList.size() );- assertEquals( "bc(4,6,4)", stack.pop().toString() );- assertEquals( "bc(8,10,8)", stack.pop().toString() );- assertEquals( "ef(11,13,11)", stack.pop().toString() );+ query.add(tq("bc"), Occur.SHOULD);+ query.add(tq("ef"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(query.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(3, stack.termList.size());+ assertEquals("bc(4,6,4)", stack.pop().toString());+ assertEquals("bc(8,10,8)", stack.pop().toString());+ assertEquals("ef(11,13,11)", stack.pop().toString());}-+public void test1PhraseB() throws Exception {makeIndexB();-- FieldQuery fq = new FieldQuery( pqF( "ab", "bb" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 4, stack.termList.size() );- assertEquals( "ab(2,4,2)", stack.pop().toString() );- assertEquals( "bb(3,5,3)", stack.pop().toString() );- assertEquals( "ab(6,8,6)", stack.pop().toString() );- assertEquals( "bb(7,9,7)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(pqF("ab", "bb"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(4, stack.termList.size());+ assertEquals("ab(2,4,2)", stack.pop().toString());+ assertEquals("bb(3,5,3)", stack.pop().toString());+ assertEquals("ab(6,8,6)", stack.pop().toString());+ assertEquals("bb(7,9,7)", stack.pop().toString());}-+private void makeIndexB() throws Exception {// 1 11 11// 01 12 23 34 45 56 67 78 89 90 01 12 (offsets)// aa|aa|ab|bb|bc|ca|ab|bb|bc|cd|de|ef// 0 1 2 3 4 5 6 7 8 9 10 11 (position)String value = "aaabbcabbcdef";-- make1dmfIndexB( value );++ make1dmfIndexB(value);}-+public void test1PhraseShortMV() throws Exception {makeIndexShortMV();-- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "d(9,10,3)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("d(9,10,3)", stack.pop().toString());}-+public void test1PhraseLongMV() throws Exception {makeIndexLongMV();-- FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 4, stack.termList.size() );- assertEquals( "search(102,108,14)", stack.pop().toString() );- assertEquals( "engines(109,116,15)", stack.pop().toString() );- assertEquals( "search(157,163,24)", stack.pop().toString() );- assertEquals( "engines(164,171,25)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(pqF("search", "engines"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(4, stack.termList.size());+ assertEquals("search(102,108,14)", stack.pop().toString());+ assertEquals("engines(109,116,15)", stack.pop().toString());+ assertEquals("search(157,163,24)", stack.pop().toString());+ assertEquals("engines(164,171,25)", stack.pop().toString());}public void test1PhraseMVB() throws Exception {makeIndexLongMVB();-- FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 4, stack.termList.size() );- assertEquals( "sp(88,90,61)", stack.pop().toString() );- assertEquals( "pe(89,91,62)", stack.pop().toString() );- assertEquals( "ee(90,92,63)", stack.pop().toString() );- assertEquals( "ed(91,93,64)", stack.pop().toString() );++ FieldQuery fq =+ new FieldQuery(+ pqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(4, stack.termList.size());+ assertEquals("sp(88,90,61)", stack.pop().toString());+ assertEquals("pe(89,91,62)", stack.pop().toString());+ assertEquals("ee(90,92,63)", stack.pop().toString());+ assertEquals("ed(91,93,64)", stack.pop().toString());}-public void testWildcard() throws Exception {makeIndexLongMV();- FieldQuery fq = new FieldQuery( new WildcardQuery (new Term(F, "th*e")), reader, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals (4, stack.termList.size());- assertEquals ("the(15,18,2)", stack.pop().toString());- assertEquals ("these(133,138,20)", stack.pop().toString());- assertEquals ("the(153,156,23)", stack.pop().toString());- assertEquals ("the(195,198,31)", stack.pop().toString());+ FieldQuery fq = new FieldQuery(new WildcardQuery(new Term(F, "th*e")), reader, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(4, stack.termList.size());+ assertEquals("the(15,18,2)", stack.pop().toString());+ assertEquals("these(133,138,20)", stack.pop().toString());+ assertEquals("the(153,156,23)", stack.pop().toString());+ assertEquals("the(195,198,31)", stack.pop().toString());}public void testTermInfoComparisonConsistency() {- TermInfo a = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 0, 1 );- TermInfo b = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 1, 1 );- TermInfo c = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 2, 1 );- TermInfo d = new TermInfo( TestUtil.randomUnicodeString(random()), 0, 0, 0, 1 );-- assertConsistentEquals( a, a );- assertConsistentEquals( b, b );- assertConsistentEquals( c, c );- assertConsistentEquals( d, d );- assertConsistentEquals( a, d );- assertConsistentLessThan( a, b );- assertConsistentLessThan( b, c );- assertConsistentLessThan( a, c );- assertConsistentLessThan( d, b );- assertConsistentLessThan( d, c );+ TermInfo a = new TermInfo(TestUtil.randomUnicodeString(random()), 0, 0, 0, 1);+ TermInfo b = new TermInfo(TestUtil.randomUnicodeString(random()), 0, 0, 1, 1);+ TermInfo c = new TermInfo(TestUtil.randomUnicodeString(random()), 0, 0, 2, 1);+ TermInfo d = new TermInfo(TestUtil.randomUnicodeString(random()), 0, 0, 0, 1);++ assertConsistentEquals(a, a);+ assertConsistentEquals(b, b);+ assertConsistentEquals(c, c);+ assertConsistentEquals(d, d);+ assertConsistentEquals(a, d);+ assertConsistentLessThan(a, b);+ assertConsistentLessThan(b, c);+ assertConsistentLessThan(a, c);+ assertConsistentLessThan(d, b);+ assertConsistentLessThan(d, c);}- private < T extends Comparable< T > > void assertConsistentEquals( T a, T b ) {- assertEquals( a, b );- assertEquals( b, a );- assertEquals( a.hashCode(), b.hashCode() );- assertEquals( 0, a.compareTo( b ) );- assertEquals( 0, b.compareTo( a ) );+ private <T extends Comparable<T>> void assertConsistentEquals(T a, T b) {+ assertEquals(a, b);+ assertEquals(b, a);+ assertEquals(a.hashCode(), b.hashCode());+ assertEquals(0, a.compareTo(b));+ assertEquals(0, b.compareTo(a));}- private < T extends Comparable< T > > void assertConsistentLessThan( T a, T b ) {- assertFalse( a.equals( b ) );- assertFalse( b.equals( a ) );- assertFalse( a.hashCode() == b.hashCode() );- assertTrue( a.compareTo( b ) < 0 );- assertTrue( b.compareTo( a ) > 0 );+ private <T extends Comparable<T>> void assertConsistentLessThan(T a, T b) {+ assertFalse(a.equals(b));+ assertFalse(b.equals(a));+ assertFalse(a.hashCode() == b.hashCode());+ assertTrue(a.compareTo(b) < 0);+ assertTrue(b.compareTo(a) > 0);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestIndexTimeSynonym.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestIndexTimeSynonym.javaindex 2bc9c787cf2..a3a2c964fb2 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestIndexTimeSynonym.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestIndexTimeSynonym.java@@ -19,213 +19,213 @@ package org.apache.lucene.search.vectorhighlight;import java.io.IOException;import org.apache.lucene.analysis.*;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;-import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.vectorhighlight.FieldTermStack.TermInfo;import org.apache.lucene.util.AttributeImpl;public class TestIndexTimeSynonym extends AbstractTestCase {-+public void testFieldTermStackIndex1wSearch1term() throws Exception {makeIndex1w();-- FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "Mac(11,20,3)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("Mac"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("Mac(11,20,3)", stack.pop().toString());}-+public void testFieldTermStackIndex1wSearch2terms() throws Exception {makeIndex1w();BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add( tq( "Mac" ), Occur.SHOULD );- bq.add( tq( "MacBook" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( bq.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );+ bq.add(tq("Mac"), Occur.SHOULD);+ bq.add(tq("MacBook"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(bq.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());TermInfo ti = stack.pop();assertEquals("Mac(11,20,3)", ti.toString());assertEquals("MacBook(11,20,3)", ti.getNext().toString());assertSame(ti, ti.getNext().getNext());}-+public void testFieldTermStackIndex1w2wSearch1term() throws Exception {makeIndex1w2w();-- FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "pc(3,5,1)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("pc"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("pc(3,5,1)", stack.pop().toString());}-+public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception {makeIndex1w2w();-- FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 2, stack.termList.size() );- assertEquals( "personal(3,5,1)", stack.pop().toString() );- assertEquals( "computer(3,5,2)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(pqF("personal", "computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(2, stack.termList.size());+ assertEquals("personal(3,5,1)", stack.pop().toString());+ assertEquals("computer(3,5,2)", stack.pop().toString());}-+public void testFieldTermStackIndex1w2wSearch1partial() throws Exception {makeIndex1w2w();-- FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "computer(3,5,2)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("computer(3,5,2)", stack.pop().toString());}-+public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception {makeIndex1w2w();BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add( tq( "pc" ), Occur.SHOULD );- bq.add( pqF( "personal", "computer" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( bq.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 2, stack.termList.size() );+ bq.add(tq("pc"), Occur.SHOULD);+ bq.add(pqF("personal", "computer"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(bq.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(2, stack.termList.size());TermInfo ti = stack.pop();- assertEquals( "pc(3,5,1)", ti.toString());- assertEquals( "personal(3,5,1)", ti.getNext().toString());+ assertEquals("pc(3,5,1)", ti.toString());+ assertEquals("personal(3,5,1)", ti.getNext().toString());assertSame(ti, ti.getNext().getNext());- assertEquals( "computer(3,5,2)", stack.pop().toString() );+ assertEquals("computer(3,5,2)", stack.pop().toString());}-+public void testFieldTermStackIndex2w1wSearch1term() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "pc(3,20,1)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("pc"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("pc(3,20,1)", stack.pop().toString());}-+public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 2, stack.termList.size() );- assertEquals( "personal(3,20,1)", stack.pop().toString() );- assertEquals( "computer(3,20,2)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(pqF("personal", "computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(2, stack.termList.size());+ assertEquals("personal(3,20,1)", stack.pop().toString());+ assertEquals("computer(3,20,2)", stack.pop().toString());}-+public void testFieldTermStackIndex2w1wSearch1partial() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 1, stack.termList.size() );- assertEquals( "computer(3,20,2)", stack.pop().toString() );++ FieldQuery fq = new FieldQuery(tq("computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(1, stack.termList.size());+ assertEquals("computer(3,20,2)", stack.pop().toString());}-+public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception {makeIndex2w1w();BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add( tq( "pc" ), Occur.SHOULD );- bq.add( pqF( "personal", "computer" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( bq.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- assertEquals( 2, stack.termList.size() );+ bq.add(tq("pc"), Occur.SHOULD);+ bq.add(pqF("personal", "computer"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(bq.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ assertEquals(2, stack.termList.size());TermInfo ti = stack.pop();assertEquals("pc(3,20,1)", ti.toString());assertEquals("personal(3,20,1)", ti.getNext().toString());assertSame(ti, ti.getNext().getNext());- assertEquals( "computer(3,20,2)", stack.pop().toString() );+ assertEquals("computer(3,20,2)", stack.pop().toString());}-+public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception {makeIndex1w2w();-- FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );++ FieldQuery fq = new FieldQuery(pqF("personal", "computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("personalcomputer(1.0)((3,5))", fpl.phraseList.get(0).toString());+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(5, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception {makeIndex1w2w();-- FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );++ FieldQuery fq = new FieldQuery(tq("computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("computer(1.0)((3,5))", fpl.phraseList.get(0).toString());+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(5, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception {makeIndex1w2w();BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add( tq( "pc" ), Occur.SHOULD );- bq.add( pqF( "personal", "computer" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( bq.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );+ bq.add(tq("pc"), Occur.SHOULD);+ bq.add(pqF("personal", "computer"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(bq.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertTrue(fpl.phraseList.get(0).toString().indexOf("(1.0)((3,5))") > 0);+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(5, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex2w1wSearch1term() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );++ FieldQuery fq = new FieldQuery(tq("pc"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("pc(1.0)((3,20))", fpl.phraseList.get(0).toString());+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(20, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );++ FieldQuery fq = new FieldQuery(pqF("personal", "computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("personalcomputer(1.0)((3,20))", fpl.phraseList.get(0).toString());+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(20, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception {makeIndex2w1w();-- FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );++ FieldQuery fq = new FieldQuery(tq("computer"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertEquals("computer(1.0)((3,20))", fpl.phraseList.get(0).toString());+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(20, fpl.phraseList.get(0).getEndOffset());}-+public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception {makeIndex2w1w();BooleanQuery.Builder bq = new BooleanQuery.Builder();- bq.add( tq( "pc" ), Occur.SHOULD );- bq.add( pqF( "personal", "computer" ), Occur.SHOULD );- FieldQuery fq = new FieldQuery( bq.build(), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- assertEquals( 1, fpl.phraseList.size() );- assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 );- assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );- assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );+ bq.add(tq("pc"), Occur.SHOULD);+ bq.add(pqF("personal", "computer"), Occur.SHOULD);+ FieldQuery fq = new FieldQuery(bq.build(), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ assertEquals(1, fpl.phraseList.size());+ assertTrue(fpl.phraseList.get(0).toString().indexOf("(1.0)((3,20))") > 0);+ assertEquals(3, fpl.phraseList.get(0).getStartOffset());+ assertEquals(20, fpl.phraseList.get(0).getEndOffset());}private void makeIndex1w() throws Exception {@@ -235,11 +235,14 @@ public class TestIndexTimeSynonym extends AbstractTestCase {// Mac// MacBook// 0 1 2 3- makeSynonymIndex( "I'll buy a Macintosh",- t("I'll",0,4),- t("buy",5,8),- t("a",9,10),- t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0));+ makeSynonymIndex(+ "I'll buy a Macintosh",+ t("I'll", 0, 4),+ t("buy", 5, 8),+ t("a", 9, 10),+ t("Macintosh", 11, 20),+ t("Mac", 11, 20, 0),+ t("MacBook", 11, 20, 0));}private void makeIndex1w2w() throws Exception {@@ -248,11 +251,14 @@ public class TestIndexTimeSynonym extends AbstractTestCase {// My pc was broken// personal computer// 0 1 2 3- makeSynonymIndex( "My pc was broken",- t("My",0,2),- t("pc",3,5),t("personal",3,5,0),t("computer",3,5),- t("was",6,9),- t("broken",10,16));+ makeSynonymIndex(+ "My pc was broken",+ t("My", 0, 2),+ t("pc", 3, 5),+ t("personal", 3, 5, 0),+ t("computer", 3, 5),+ t("was", 6, 9),+ t("broken", 10, 16));}private void makeIndex2w1w() throws Exception {@@ -261,54 +267,60 @@ public class TestIndexTimeSynonym extends AbstractTestCase {// My personal computer was broken// pc// 0 1 2 3 4- makeSynonymIndex( "My personal computer was broken",- t("My",0,2),- t("personal",3,20),t("pc",3,20,0),t("computer",3,20),- t("was",21,24),- t("broken",25,31));+ makeSynonymIndex(+ "My personal computer was broken",+ t("My", 0, 2),+ t("personal", 3, 20),+ t("pc", 3, 20, 0),+ t("computer", 3, 20),+ t("was", 21, 24),+ t("broken", 25, 31));}-- void makeSynonymIndex( String value, Token... tokens ) throws Exception {- Analyzer analyzer = new TokenArrayAnalyzer( tokens );- make1dmfIndex( analyzer, value );++ void makeSynonymIndex(String value, Token... tokens) throws Exception {+ Analyzer analyzer = new TokenArrayAnalyzer(tokens);+ make1dmfIndex(analyzer, value);}- public static Token t( String text, int startOffset, int endOffset ){- return t( text, startOffset, endOffset, 1 );+ public static Token t(String text, int startOffset, int endOffset) {+ return t(text, startOffset, endOffset, 1);}-- public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){- Token token = new Token( text, startOffset, endOffset );- token.setPositionIncrement( positionIncrement );++ public static Token t(String text, int startOffset, int endOffset, int positionIncrement) {+ Token token = new Token(text, startOffset, endOffset);+ token.setPositionIncrement(positionIncrement);return token;}-+public static final class TokenArrayAnalyzer extends Analyzer {final Token[] tokens;+public TokenArrayAnalyzer(Token... tokens) {this.tokens = tokens;}-+@Overridepublic TokenStreamComponents createComponents(String fieldName) {- Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {- final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);- int p = 0;-- @Override- public boolean incrementToken() {- if( p >= tokens.length ) return false;- clearAttributes();- tokens[p++].copyTo(reusableToken);- return true;- }-- @Override- public void reset() throws IOException {- super.reset();- this.p = 0;- }- };+ Tokenizer ts =+ new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {+ final AttributeImpl reusableToken =+ (AttributeImpl) addAttribute(CharTermAttribute.class);+ int p = 0;++ @Override+ public boolean incrementToken() {+ if (p >= tokens.length) return false;+ clearAttributes();+ tokens[p++].copyTo(reusableToken);+ return true;+ }++ @Override+ public void reset() throws IOException {+ super.reset();+ this.p = 0;+ }+ };return new TokenStreamComponents(ts);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestScoreOrderFragmentsBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestScoreOrderFragmentsBuilder.javaindex d964febaa45..1d5c64314bc 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestScoreOrderFragmentsBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestScoreOrderFragmentsBuilder.java@@ -23,27 +23,27 @@ import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;public class TestScoreOrderFragmentsBuilder extends AbstractTestCase {-+public void test3Frags() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();query.add(new TermQuery(new Term(F, "a")), BooleanClause.Occur.SHOULD);query.add(new TermQuery(new Term(F, "c")), BooleanClause.Occur.SHOULD);- FieldFragList ffl = ffl(query.build(), "a b b b b b b b b b b b a b a b b b b b c a a b b" );+ FieldFragList ffl = ffl(query.build(), "a b b b b b b b b b b b a b a b b b b b c a a b b");ScoreOrderFragmentsBuilder sofb = new ScoreOrderFragmentsBuilder();- String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );- assertEquals( 3, f.length );+ String[] f = sofb.createFragments(reader, 0, F, ffl, 3);+ assertEquals(3, f.length);// check score order- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );- assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );- assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );+ assertEquals("<b>c</b> <b>a</b> <b>a</b> b b", f[0]);+ assertEquals("b b <b>a</b> b <b>a</b> b b b b b c", f[1]);+ assertEquals("<b>a</b> b b b b b b b b b b", f[2]);}- private FieldFragList ffl(Query query, String indexValue ) throws Exception {- make1d1fIndex( indexValue );- FieldQuery fq = new FieldQuery( query, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- return new SimpleFragListBuilder().createFieldFragList( fpl, 20 );+ private FieldFragList ffl(Query query, String indexValue) throws Exception {+ make1d1fIndex(indexValue);+ FieldQuery fq = new FieldQuery(query, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ return new SimpleFragListBuilder().createFieldFragList(fpl, 20);}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleBoundaryScanner.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleBoundaryScanner.javaindex 0e395c4b01a..7d01e5061c7 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleBoundaryScanner.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleBoundaryScanner.java@@ -20,22 +20,22 @@ import org.apache.lucene.util.LuceneTestCase;public class TestSimpleBoundaryScanner extends LuceneTestCase {static final String TEXT =- "Apache Lucene(TM) is a high-performance, full-featured\ntext search engine library written entirely in Java.";+ "Apache Lucene(TM) is a high-performance, full-featured\ntext search engine library written entirely in Java.";public void testFindStartOffset() throws Exception {StringBuilder text = new StringBuilder(TEXT);BoundaryScanner scanner = new SimpleBoundaryScanner();-+// test out of rangeint start = TEXT.length() + 1;assertEquals(start, scanner.findStartOffset(text, start));start = 0;assertEquals(start, scanner.findStartOffset(text, start));-+start = TEXT.indexOf("formance");int expected = TEXT.indexOf("high-performance");assertEquals(expected, scanner.findStartOffset(text, start));-+start = TEXT.indexOf("che");expected = TEXT.indexOf("Apache");assertEquals(expected, scanner.findStartOffset(text, start));@@ -44,13 +44,13 @@ public class TestSimpleBoundaryScanner extends LuceneTestCase {public void testFindEndOffset() throws Exception {StringBuilder text = new StringBuilder(TEXT);BoundaryScanner scanner = new SimpleBoundaryScanner();-+// test out of rangeint start = TEXT.length() + 1;assertEquals(start, scanner.findEndOffset(text, start));start = -1;assertEquals(start, scanner.findEndOffset(text, start));-+start = TEXT.indexOf("full-");int expected = TEXT.indexOf("\ntext");assertEquals(expected, scanner.findEndOffset(text, start));diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragListBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragListBuilder.javaindex 66a2a0cb325..6b8de7d19d6 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragListBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragListBuilder.java@@ -20,78 +20,93 @@ import org.apache.lucene.index.Term;import org.apache.lucene.search.*;public class TestSimpleFragListBuilder extends AbstractTestCase {-+public void testNullFieldFragList() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "b c d" ), 100 );- assertEquals( 0, ffl.getFragInfos().size() );+ FieldFragList ffl =+ sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "b c d"), 100);+ assertEquals(0, ffl.getFragInfos().size());}-+public void testTooSmallFragSize() throws Exception {- expectThrows(IllegalArgumentException.class, () -> {- SimpleFragListBuilder sflb = new SimpleFragListBuilder();- sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "b c d"), sflb.minFragCharSize - 1);- });+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ SimpleFragListBuilder sflb = new SimpleFragListBuilder();+ sflb.createFieldFragList(+ fpl(new TermQuery(new Term(F, "a")), "b c d"), sflb.minFragCharSize - 1);+ });}-+public void testSmallerFragSizeThanTermQuery() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "abcdefghijklmnopqrs")), "abcdefghijklmnopqrs" ), sflb.minFragCharSize );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(abcdefghijklmnopqrs((0,19)))/1.0(0,19)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl =+ sflb.createFieldFragList(+ fpl(new TermQuery(new Term(F, "abcdefghijklmnopqrs")), "abcdefghijklmnopqrs"),+ sflb.minFragCharSize);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals(+ "subInfos=(abcdefghijklmnopqrs((0,19)))/1.0(0,19)", ffl.getFragInfos().get(0).toString());}-+public void testSmallerFragSizeThanPhraseQuery() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();PhraseQuery phraseQuery = new PhraseQuery(F, "abcdefgh", "jklmnopqrs");- FieldFragList ffl = sflb.createFieldFragList( fpl(phraseQuery, "abcdefgh jklmnopqrs" ), sflb.minFragCharSize );- assertEquals( 1, ffl.getFragInfos().size() );- if (VERBOSE) System.out.println( ffl.getFragInfos().get( 0 ).toString() );- assertEquals( "subInfos=(abcdefghjklmnopqrs((0,21)))/1.0(0,21)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl =+ sflb.createFieldFragList(fpl(phraseQuery, "abcdefgh jklmnopqrs"), sflb.minFragCharSize);+ assertEquals(1, ffl.getFragInfos().size());+ if (VERBOSE) System.out.println(ffl.getFragInfos().get(0).toString());+ assertEquals(+ "subInfos=(abcdefghjklmnopqrs((0,21)))/1.0(0,21)", ffl.getFragInfos().get(0).toString());}-+public void test1TermIndex() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a" ), 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1)))/1.0(0,100)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "a"), 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1)))/1.0(0,100)", ffl.getFragInfos().get(0).toString());}-+public void test2TermsIndex1Frag() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a a" ), 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1))a((2,3)))/2.0(0,100)", ffl.getFragInfos().get( 0 ).toString() );-- ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b a" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1))a((18,19)))/2.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );-- ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "b b b b a b b b b a" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((8,9))a((18,19)))/2.0(4,24)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "a a"), 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1))a((2,3)))/2.0(0,100)", ffl.getFragInfos().get(0).toString());++ ffl = sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b a"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1))a((18,19)))/2.0(0,20)", ffl.getFragInfos().get(0).toString());++ ffl = sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "b b b b a b b b b a"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((8,9))a((18,19)))/2.0(4,24)", ffl.getFragInfos().get(0).toString());}-+public void test2TermsIndex2Frags() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b b b a" ), 20 );- assertEquals( 2, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );- assertEquals( "subInfos=(a((28,29)))/1.0(20,40)", ffl.getFragInfos().get( 1 ).toString() );-- ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b b a" ), 20 );- assertEquals( 2, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );- assertEquals( "subInfos=(a((26,27)))/1.0(20,40)", ffl.getFragInfos().get( 1 ).toString() );-- ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b a" ), 20 );- assertEquals( 2, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );- assertEquals( "subInfos=(a((20,21)))/1.0(20,40)", ffl.getFragInfos().get( 1 ).toString() );+ FieldFragList ffl =+ sflb.createFieldFragList(+ fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b b b a"), 20);+ assertEquals(2, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());+ assertEquals("subInfos=(a((28,29)))/1.0(20,40)", ffl.getFragInfos().get(1).toString());++ ffl =+ sflb.createFieldFragList(+ fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b b a"), 20);+ assertEquals(2, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());+ assertEquals("subInfos=(a((26,27)))/1.0(20,40)", ffl.getFragInfos().get(1).toString());++ ffl =+ sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b a"), 20);+ assertEquals(2, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());+ assertEquals("subInfos=(a((20,21)))/1.0(20,40)", ffl.getFragInfos().get(1).toString());}-+public void test2TermsQuery() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();@@ -99,84 +114,88 @@ public class TestSimpleFragListBuilder extends AbstractTestCase {booleanQuery.add(new TermQuery(new Term(F, "a")), BooleanClause.Occur.SHOULD);booleanQuery.add(new TermQuery(new Term(F, "b")), BooleanClause.Occur.SHOULD);- FieldFragList ffl = sflb.createFieldFragList( fpl(booleanQuery.build(), "c d e" ), 20 );- assertEquals( 0, ffl.getFragInfos().size() );+ FieldFragList ffl = sflb.createFieldFragList(fpl(booleanQuery.build(), "c d e"), 20);+ assertEquals(0, ffl.getFragInfos().size());- ffl = sflb.createFieldFragList( fpl(booleanQuery.build(), "d b c" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(b((2,3)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );+ ffl = sflb.createFieldFragList(fpl(booleanQuery.build(), "d b c"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(b((2,3)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());- ffl = sflb.createFieldFragList( fpl(booleanQuery.build(), "a b c" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1))b((2,3)))/2.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );+ ffl = sflb.createFieldFragList(fpl(booleanQuery.build(), "a b c"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1))b((2,3)))/2.0(0,20)", ffl.getFragInfos().get(0).toString());}-+public void testPhraseQuery() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();PhraseQuery phraseQuery = new PhraseQuery(F, "a", "b");- FieldFragList ffl = sflb.createFieldFragList( fpl(phraseQuery, "c d e" ), 20 );- assertEquals( 0, ffl.getFragInfos().size() );+ FieldFragList ffl = sflb.createFieldFragList(fpl(phraseQuery, "c d e"), 20);+ assertEquals(0, ffl.getFragInfos().size());- ffl = sflb.createFieldFragList( fpl(phraseQuery, "a c b" ), 20 );- assertEquals( 0, ffl.getFragInfos().size() );+ ffl = sflb.createFieldFragList(fpl(phraseQuery, "a c b"), 20);+ assertEquals(0, ffl.getFragInfos().size());- ffl = sflb.createFieldFragList( fpl(phraseQuery, "a b c" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(ab((0,3)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );+ ffl = sflb.createFieldFragList(fpl(phraseQuery, "a b c"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(ab((0,3)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());}-+public void testPhraseQuerySlop() throws Exception {SimpleFragListBuilder sflb = new SimpleFragListBuilder();PhraseQuery phraseQuery = new PhraseQuery(1, F, "a", "b");- FieldFragList ffl = sflb.createFieldFragList( fpl(phraseQuery, "a c b" ), 20 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(ab((0,1)(4,5)))/1.0(0,20)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl(phraseQuery, "a c b"), 20);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(ab((0,1)(4,5)))/1.0(0,20)", ffl.getFragInfos().get(0).toString());}- private FieldPhraseList fpl(Query query, String indexValue ) throws Exception {- make1d1fIndex( indexValue );- FieldQuery fq = new FieldQuery( query, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- return new FieldPhraseList( stack, fq );+ private FieldPhraseList fpl(Query query, String indexValue) throws Exception {+ make1d1fIndex(indexValue);+ FieldQuery fq = new FieldQuery(query, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ return new FieldPhraseList(stack, fq);}-+public void test1PhraseShortMV() throws Exception {makeIndexShortMV();- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(d((9,10)))/1.0(0,100)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(d((9,10)))/1.0(0,100)", ffl.getFragInfos().get(0).toString());}-+public void test1PhraseLongMV() throws Exception {makeIndexLongMV();- FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(pqF("search", "engines"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(searchengines((102,116))searchengines((157,171)))/2.0(87,187)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals(+ "subInfos=(searchengines((102,116))searchengines((157,171)))/2.0(87,187)",+ ffl.getFragInfos().get(0).toString());}public void test1PhraseLongMVB() throws Exception {makeIndexLongMVB();- FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq =+ new FieldQuery(+ pqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(sppeeeed((88,93)))/1.0(41,141)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(sppeeeed((88,93)))/1.0(41,141)", ffl.getFragInfos().get(0).toString());}}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragmentsBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragmentsBuilder.javaindex 9a4e879b907..c3ef1bb5c50 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragmentsBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSimpleFragmentsBuilder.java@@ -16,6 +16,12 @@*/package org.apache.lucene.search.vectorhighlight;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;import org.apache.lucene.analysis.MockAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -36,185 +42,187 @@ import org.apache.lucene.search.highlight.SimpleHTMLEncoder;import org.apache.lucene.store.Directory;import org.apache.lucene.util.TestUtil;-import java.util.ArrayList;-import java.util.HashMap;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.Set;-public class TestSimpleFragmentsBuilder extends AbstractTestCase {-+public void test1TermIndex() throws Exception {- FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" );+ FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a");SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );+ assertEquals("<b>a</b>", sfb.createFragment(reader, 0, F, ffl));// change tags- sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );- assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );+ sfb = new SimpleFragmentsBuilder(new String[] {"["}, new String[] {"]"});+ assertEquals("[a]", sfb.createFragment(reader, 0, F, ffl));}-+public void test2Frags() throws Exception {- FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b a b a b" );+ FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b a b a b");SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- String[] f = sfb.createFragments( reader, 0, F, ffl, 3 );+ String[] f = sfb.createFragments(reader, 0, F, ffl, 3);// 3 snippets requested, but should be 2- assertEquals( 2, f.length );- assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );- assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );+ assertEquals(2, f.length);+ assertEquals("<b>a</b> b b b b b b b b b b", f[0]);+ assertEquals("b b <b>a</b> b <b>a</b> b", f[1]);}-+public void test3Frags() throws Exception {BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();booleanQuery.add(new TermQuery(new Term(F, "a")), BooleanClause.Occur.SHOULD);booleanQuery.add(new TermQuery(new Term(F, "c")), BooleanClause.Occur.SHOULD);-- FieldFragList ffl = ffl(booleanQuery.build(), "a b b b b b b b b b b b a b a b b b b b c a a b b" );++ FieldFragList ffl =+ ffl(booleanQuery.build(), "a b b b b b b b b b b b a b a b b b b b c a a b b");SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- String[] f = sfb.createFragments( reader, 0, F, ffl, 3 );- assertEquals( 3, f.length );- assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );- assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );+ String[] f = sfb.createFragments(reader, 0, F, ffl, 3);+ assertEquals(3, f.length);+ assertEquals("<b>a</b> b b b b b b b b b b", f[0]);+ assertEquals("b b <b>a</b> b <b>a</b> b b b b b c", f[1]);+ assertEquals("<b>c</b> <b>a</b> <b>a</b> b b", f[2]);}-+public void testTagsAndEncoder() throws Exception {- FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "<h1> a </h1>" );+ FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "<h1> a </h1>");SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- String[] preTags = { "[" };- String[] postTags = { "]" };- assertEquals( "<h1> [a] </h1>",- sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );+ String[] preTags = {"["};+ String[] postTags = {"]"};+ assertEquals(+ "<h1> [a] </h1>",+ sfb.createFragment(reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder()));}- private FieldFragList ffl(Query query, String indexValue ) throws Exception {- make1d1fIndex( indexValue );- FieldQuery fq = new FieldQuery( query, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );- return new SimpleFragListBuilder().createFieldFragList( fpl, 20 );+ private FieldFragList ffl(Query query, String indexValue) throws Exception {+ make1d1fIndex(indexValue);+ FieldQuery fq = new FieldQuery(query, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);+ return new SimpleFragListBuilder().createFieldFragList(fpl, 20);}-+public void test1PhraseShortMV() throws Exception {makeIndexShortMV();- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();// Should we probably be trimming?- assertEquals( " a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );+ assertEquals(" a b c <b>d</b> e", sfb.createFragment(reader, 0, F, ffl));}-+public void test1PhraseLongMV() throws Exception {makeIndexLongMV();- FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(pqF("search", "engines"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- assertEquals( "customization: The most <b>search engines</b> use only one of these methods. Even the <b>search engines</b> that says they can",- sfb.createFragment( reader, 0, F, ffl ) );+ assertEquals(+ "customization: The most <b>search engines</b> use only one of these methods. Even the <b>search engines</b> that says they can",+ sfb.createFragment(reader, 0, F, ffl));}public void test1PhraseLongMVB() throws Exception {makeIndexLongMVB();- FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed"- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq =+ new FieldQuery(+ pqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- assertEquals( "additional hardware. \nWhen you talk about processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );+ assertEquals(+ "additional hardware. \nWhen you talk about processing <b>speed</b>, the",+ sfb.createFragment(reader, 0, F, ffl));}-+public void testUnstoredField() throws Exception {makeUnstoredIndex();- FieldQuery fq = new FieldQuery( tq( "aaa" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("aaa"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- assertNull( sfb.createFragment( reader, 0, F, ffl ) );+ assertNull(sfb.createFragment(reader, 0, F, ffl));}-+protected void makeUnstoredIndex() throws Exception {- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzerW).setOpenMode(OpenMode.CREATE));+ IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(analyzerW).setOpenMode(OpenMode.CREATE));Document doc = new Document();FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);customType.setStoreTermVectors(true);customType.setStoreTermVectorOffsets(true);customType.setStoreTermVectorPositions(true);- doc.add( new Field( F, "aaa", customType) );- //doc.add( new Field( F, "aaa", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );- writer.addDocument( doc );+ doc.add(new Field(F, "aaa", customType));+ // doc.add( new Field( F, "aaa", Store.NO, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS )+ // );+ writer.addDocument(doc);writer.close();if (reader != null) reader.close();reader = DirectoryReader.open(dir);}-+public void test1StrMV() throws Exception {makeIndexStrMV();- FieldQuery fq = new FieldQuery( tq( "defg" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("defg"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- sfb.setMultiValuedSeparator( '/' );- assertEquals( "abc/<b>defg</b>/hijkl", sfb.createFragment( reader, 0, F, ffl ) );+ sfb.setMultiValuedSeparator('/');+ assertEquals("abc/<b>defg</b>/hijkl", sfb.createFragment(reader, 0, F, ffl));}-+public void testMVSeparator() throws Exception {makeIndexShortMV();- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();- sfb.setMultiValuedSeparator( '/' );- assertEquals( "//a b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );+ sfb.setMultiValuedSeparator('/');+ assertEquals("//a b c//<b>d</b> e", sfb.createFragment(reader, 0, F, ffl));}public void testDiscreteMultiValueHighlighting() throws Exception {makeIndexShortMV();- FieldQuery fq = new FieldQuery( tq( "d" ), true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(tq("d"), true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);SimpleFragListBuilder sflb = new SimpleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );+ FieldFragList ffl = sflb.createFieldFragList(fpl, 100);SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();sfb.setDiscreteMultiValueHighlighting(true);- assertEquals( "<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );+ assertEquals("<b>d</b> e", sfb.createFragment(reader, 0, F, ffl));make1dmfIndex("some text to highlight", "highlight other text");- fq = new FieldQuery( tq( "text" ), true, true );- stack = new FieldTermStack( reader, 0, F, fq );- fpl = new FieldPhraseList( stack, fq );+ fq = new FieldQuery(tq("text"), true, true);+ stack = new FieldTermStack(reader, 0, F, fq);+ fpl = new FieldPhraseList(stack, fq);sflb = new SimpleFragListBuilder();- ffl = sflb.createFieldFragList( fpl, 32 );+ ffl = sflb.createFieldFragList(fpl, 32);String[] result = sfb.createFragments(reader, 0, F, ffl, 3);assertEquals(2, result.length);assertEquals("some <b>text</b> to highlight", result[0]);assertEquals("highlight other <b>text</b>", result[1]);- fq = new FieldQuery( tq( "highlight" ), true, true );- stack = new FieldTermStack( reader, 0, F, fq );- fpl = new FieldPhraseList( stack, fq );+ fq = new FieldQuery(tq("highlight"), true, true);+ stack = new FieldTermStack(reader, 0, F, fq);+ fpl = new FieldPhraseList(stack, fq);sflb = new SimpleFragListBuilder();- ffl = sflb.createFieldFragList( fpl, 32 );+ ffl = sflb.createFieldFragList(fpl, 32);result = sfb.createFragments(reader, 0, F, ffl, 3);assertEquals(2, result.length);assertEquals("text to <b>highlight</b>", result[0]);@@ -232,10 +240,11 @@ public class TestSimpleFragmentsBuilder extends AbstractTestCase {}Directory dir = newDirectory();- RandomIndexWriter writer = new RandomIndexWriter(- random(),- dir,- newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));+ RandomIndexWriter writer =+ new RandomIndexWriter(+ random(),+ dir,+ newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));FieldType customType = new FieldType(TextField.TYPE_STORED);customType.setStoreTermVectors(true);@@ -315,7 +324,8 @@ public class TestSimpleFragmentsBuilder extends AbstractTestCase {}}- private String getRandomValue(String[] randomValues, Map<String, Set<Integer>> valueToDocId, int docId) {+ private String getRandomValue(+ String[] randomValues, Map<String, Set<Integer>> valueToDocId, int docId) {String value = randomValues[random().nextInt(randomValues.length)];if (!valueToDocId.containsKey(value)) {valueToDocId.put(value, new HashSet<Integer>());@@ -332,5 +342,4 @@ public class TestSimpleFragmentsBuilder extends AbstractTestCase {this.fieldValues = fieldValues;}}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSingleFragListBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSingleFragListBuilder.javaindex 1286660c285..41b4b5882c5 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSingleFragListBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestSingleFragListBuilder.java@@ -21,32 +21,43 @@ import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;public class TestSingleFragListBuilder extends AbstractTestCase {-+public void testNullFieldFragList() throws Exception {SingleFragListBuilder sflb = new SingleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "b c d" ), 100 );- assertEquals( 0, ffl.getFragInfos().size() );+ FieldFragList ffl =+ sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "b c d"), 100);+ assertEquals(0, ffl.getFragInfos().size());}-+public void testShortFieldFragList() throws Exception {SingleFragListBuilder sflb = new SingleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b c d" ), 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1)))/1.0(0,2147483647)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl =+ sflb.createFieldFragList(fpl(new TermQuery(new Term(F, "a")), "a b c d"), 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals("subInfos=(a((0,1)))/1.0(0,2147483647)", ffl.getFragInfos().get(0).toString());}-+public void testLongFieldFragList() throws Exception {SingleFragListBuilder sflb = new SingleFragListBuilder();- FieldFragList ffl = sflb.createFieldFragList( fpl(new TermQuery(new Term(F, "a")), "a b c d", "a b c d e f g h i", "j k l m n o p q r s t u v w x y z a b c", "d e f g" ), 100 );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( "subInfos=(a((0,1))a((8,9))a((60,61)))/3.0(0,2147483647)", ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl =+ sflb.createFieldFragList(+ fpl(+ new TermQuery(new Term(F, "a")),+ "a b c d",+ "a b c d e f g h i",+ "j k l m n o p q r s t u v w x y z a b c",+ "d e f g"),+ 100);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals(+ "subInfos=(a((0,1))a((8,9))a((60,61)))/3.0(0,2147483647)",+ ffl.getFragInfos().get(0).toString());}- private FieldPhraseList fpl(Query query, String... indexValues ) throws Exception {- make1dmfIndex( indexValues );- FieldQuery fq = new FieldQuery( query, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- return new FieldPhraseList( stack, fq );+ private FieldPhraseList fpl(Query query, String... indexValues) throws Exception {+ make1dmfIndex(indexValues);+ FieldQuery fq = new FieldQuery(query, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ return new FieldPhraseList(stack, fq);}-}diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestWeightedFragListBuilder.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestWeightedFragListBuilder.javaindex 4917ce43640..9fcf4d67298 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestWeightedFragListBuilder.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestWeightedFragListBuilder.java@@ -24,40 +24,41 @@ import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.Spublic class TestWeightedFragListBuilder extends AbstractTestCase {public void test2WeightedFragList() throws Exception {- testCase( pqF( "the", "both" ), 100,- "subInfos=(theboth((195,203)))/0.8679108(149,249)",- 0.8679108 );+ testCase(+ pqF("the", "both"), 100, "subInfos=(theboth((195,203)))/0.8679108(149,249)", 0.8679108);}public void test2SubInfos() throws Exception {BooleanQuery.Builder query = new BooleanQuery.Builder();- query.add( pqF( "the", "both" ), Occur.MUST );- query.add( tq( "examples" ), Occur.MUST );+ query.add(pqF("the", "both"), Occur.MUST);+ query.add(tq("examples"), Occur.MUST);- testCase( query.build(), 1000,+ testCase(+ query.build(),+ 1000,"subInfos=(examples((19,27))examples((66,74))theboth((195,203)))/1.8411169(0,1000)",- 1.8411169 );+ 1.8411169);}- private void testCase( Query query, int fragCharSize, String expectedFragInfo,- double expectedTotalSubInfoBoost ) throws Exception {+ private void testCase(+ Query query, int fragCharSize, String expectedFragInfo, double expectedTotalSubInfoBoost)+ throws Exception {makeIndexLongMV();- FieldQuery fq = new FieldQuery( query, true, true );- FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );- FieldPhraseList fpl = new FieldPhraseList( stack, fq );+ FieldQuery fq = new FieldQuery(query, true, true);+ FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);WeightedFragListBuilder wflb = new WeightedFragListBuilder();- FieldFragList ffl = wflb.createFieldFragList( fpl, fragCharSize );- assertEquals( 1, ffl.getFragInfos().size() );- assertEquals( expectedFragInfo, ffl.getFragInfos().get( 0 ).toString() );+ FieldFragList ffl = wflb.createFieldFragList(fpl, fragCharSize);+ assertEquals(1, ffl.getFragInfos().size());+ assertEquals(expectedFragInfo, ffl.getFragInfos().get(0).toString());float totalSubInfoBoost = 0;- for ( WeightedFragInfo info : ffl.getFragInfos() ) {- for ( SubInfo subInfo : info.getSubInfos() ) {+ for (WeightedFragInfo info : ffl.getFragInfos()) {+ for (SubInfo subInfo : info.getSubInfos()) {totalSubInfoBoost += subInfo.getBoost();}}- assertEquals( expectedTotalSubInfoBoost, totalSubInfoBoost, .0000001 );+ assertEquals(expectedTotalSubInfoBoost, totalSubInfoBoost, .0000001);}-}