Apache Solr open-source search software
SOLR-18362: replace org.apache.solr.spelling.Token with SpellCheckToken (#4812)
Serhiy Bzhezytskyybdb16c9706cf
diff --git a/changelog/unreleased/SOLR-18362-remove-spelling-token.yml b/changelog/unreleased/SOLR-18362-remove-spelling-token.ymlnew file mode 100644index 00000000000..468ed975392--- /dev/null+++ b/changelog/unreleased/SOLR-18362-remove-spelling-token.yml@@ -0,0 +1,15 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: >+ The spellchecker API no longer passes terms around as `org.apache.solr.spelling.Token`, which is+ removed. Custom plugins in the `org.apache.solr.spelling` package need updating:+ `QueryConverter.convert(String)` now returns a Lucene `TokenStream` rather than a+ `Collection<Token>`, `SpellingOptions.tokens` is a `List<SpellCheckToken>` rather than a+ `Collection<Token>`, and `SpellingResult` is keyed by the new immutable record `SpellCheckToken`+ wherever it previously took a `Token`. Behaviour of the shipped spellcheckers and of the `/spell`+ request handler is unchanged.+type: removed+authors:+ - name: Serhiy Bzhezytskyy+links:+ - name: SOLR-18362+ url: https://issues.apache.org/jira/browse/SOLR-18362diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.javaindex 98af4ed46dc..0221f720f4d 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java@@ -20,7 +20,6 @@ import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.Arrays;-import java.util.Collection;import java.util.Collections;import java.util.HashMap;import java.util.HashSet;@@ -32,12 +31,6 @@ import java.util.concurrent.ConcurrentHashMap;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.core.WhitespaceAnalyzer;-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.ExitableDirectoryReader;import org.apache.lucene.search.Query;@@ -69,10 +62,10 @@ import org.apache.solr.spelling.QueryConverter;import org.apache.solr.spelling.SolrSpellChecker;import org.apache.solr.spelling.SpellCheckCollation;import org.apache.solr.spelling.SpellCheckCollator;+import org.apache.solr.spelling.SpellCheckToken;import org.apache.solr.spelling.SpellingOptions;import org.apache.solr.spelling.SpellingQueryConverter;import org.apache.solr.spelling.SpellingResult;-import org.apache.solr.spelling.Token;import org.apache.solr.util.SolrResponseUtil;import org.apache.solr.util.plugin.SolrCoreAware;import org.slf4j.Logger;@@ -143,20 +136,23 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwarSolrSpellChecker spellChecker = getSpellChecker(params);if (spellChecker != null) {- Collection<Token> tokens;String q = params.get(SPELLCHECK_Q);+ // the query is analyzed once, here: every spellchecker reads every token, and+ // ConjunctionSolrSpellChecker hands the same options to each of its children, so one list+ // serves them all.+ final List<SpellCheckToken> tokens;if (q != null) {// we have a spell check param, tokenize it with the query analyzer applicable for this// spellchecker- tokens = getTokens(q, spellChecker.getQueryAnalyzer());+ tokens = SpellCheckToken.drain(getTokens(q, spellChecker.getQueryAnalyzer()));} else {q = rb.getQueryString();if (q == null) {q = params.get(CommonParams.Q);}- tokens = queryConverter.convert(q);+ tokens = SpellCheckToken.drain(queryConverter.convert(q));}- if (tokens != null && tokens.isEmpty() == false) {+ if (!tokens.isEmpty()) {int count = params.getInt(SPELLCHECK_COUNT, 1);boolean onlyMorePopular =params.getBool(SPELLCHECK_ONLY_MORE_POPULAR, DEFAULT_ONLY_MORE_POPULAR);@@ -347,10 +343,10 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar}private void addOriginalTermsToResponse(- NamedList<Object> response, Collection<Token> originalTerms) {- List<String> originalTermStr = new ArrayList<String>();- for (Token t : originalTerms) {- originalTermStr.add(t.toString());+ NamedList<Object> response, List<SpellCheckToken> originalTerms) {+ List<String> originalTermStr = new ArrayList<>(originalTerms.size());+ for (SpellCheckToken token : originalTerms) {+ originalTermStr.add(token.text());}response.add("originalTerms", originalTermStr);}@@ -580,32 +576,9 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar}}- private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException {- Collection<Token> result = new ArrayList<>();+ private TokenStream getTokens(String q, Analyzer analyzer) {assert analyzer != null;- try (TokenStream ts = analyzer.tokenStream("", q)) {- ts.reset();- // TODO: support custom attributes- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);- TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);- FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);- PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);- PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);-- while (ts.incrementToken()) {- Token token = new Token();- token.copyBuffer(termAtt.buffer(), 0, termAtt.length());- token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());- token.setType(typeAtt.type());- token.setFlags(flagsAtt.getFlags());- token.setPayload(payloadAtt.getPayload());- token.setPositionIncrement(posIncAtt.getPositionIncrement());- result.add(token);- }- ts.end();- return result;- }+ return analyzer.tokenStream("", q);}protected SolrSpellChecker getSpellChecker(SolrParams params) {@@ -658,13 +631,15 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwarString origQuery,boolean extendedResults) {NamedList<Object> result = new NamedList<>();- Map<Token, LinkedHashMap<String, Integer>> suggestions = spellingResult.getSuggestions();+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions =+ spellingResult.getSuggestions();boolean hasFreqInfo = spellingResult.hasTokenFrequencyInfo();boolean hasSuggestions = false;boolean hasZeroFrequencyToken = false;- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry : suggestions.entrySet()) {- Token inputToken = entry.getKey();- String tokenString = new String(inputToken.buffer(), 0, inputToken.length());+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :+ suggestions.entrySet()) {+ SpellCheckToken inputToken = entry.getKey();+ String tokenString = inputToken.text();Map<String, Integer> theSuggestions = new LinkedHashMap<>(entry.getValue());theSuggestions.keySet().removeIf(sug -> sug.equals(tokenString));if (theSuggestions.size() > 0) {diff --git a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.javaindex 25cb239b2e0..284df9a71e8 100644--- a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java+++ b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java@@ -132,19 +132,19 @@ public abstract class AbstractLuceneSpellChecker extends SolrSpellChecker {@Overridepublic SpellingResult getSuggestions(SpellingOptions options) throws IOException {- SpellingResult result = new SpellingResult(options.tokens);+ SpellingResult result = new SpellingResult();IndexReader reader = determineReader(options.reader);Term term = field != null ? new Term(field, "") : null;float theAccuracy =(options.accuracy == Float.MIN_VALUE) ? spellChecker.getAccuracy() : options.accuracy;int count = Math.max(options.count, AbstractLuceneSpellChecker.DEFAULT_SUGGESTION_COUNT);- for (Token token : options.tokens) {- if (token.length() == 0) {+ for (SpellCheckToken token : options.tokens) {+ String tokenText = token.text();+ if (tokenText.isEmpty()) {result.add(token, List.of());continue;}- String tokenText = new String(token.buffer(), 0, token.length());term = new Term(field, tokenText);int docFreq = 0;if (reader != null) {diff --git a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.javaindex dc67b47cb17..077464bd0aa 100644--- a/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java+++ b/solr/core/src/java/org/apache/solr/spelling/ConjunctionSolrSpellChecker.java@@ -131,13 +131,14 @@ public class ConjunctionSolrSpellChecker extends SolrSpellChecker {// TODO: This just interleaves the results. In the future, we might want to let users give each// checker its own weight and use that in combination to score & frequency to sort the results ?private SpellingResult mergeCheckers(SpellingResult[] results, int numSug) {- Map<Token, Integer> combinedTokenFrequency = new HashMap<>();- Map<Token, List<LinkedHashMap<String, Integer>>> allSuggestions = new LinkedHashMap<>();+ Map<SpellCheckToken, Integer> combinedTokenFrequency = new HashMap<>();+ Map<SpellCheckToken, List<LinkedHashMap<String, Integer>>> allSuggestions =+ new LinkedHashMap<>();for (SpellingResult result : results) {if (result.getTokenFrequency() != null) {combinedTokenFrequency.putAll(result.getTokenFrequency());}- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry :+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :result.getSuggestions().entrySet()) {List<LinkedHashMap<String, Integer>> allForThisToken = allSuggestions.get(entry.getKey());if (allForThisToken == null) {@@ -148,8 +149,9 @@ public class ConjunctionSolrSpellChecker extends SolrSpellChecker {}}SpellingResult combinedResult = new SpellingResult();- for (Map.Entry<Token, List<LinkedHashMap<String, Integer>>> entry : allSuggestions.entrySet()) {- Token original = entry.getKey();+ for (Map.Entry<SpellCheckToken, List<LinkedHashMap<String, Integer>>> entry :+ allSuggestions.entrySet()) {+ SpellCheckToken original = entry.getKey();List<Iterator<Map.Entry<String, Integer>>> corrIters =new ArrayList<>(entry.getValue().size());for (LinkedHashMap<String, Integer> corrections : entry.getValue()) {diff --git a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.javaindex acbb8828316..09397ce39cf 100644--- a/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java+++ b/solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java@@ -183,18 +183,16 @@ public class DirectSolrSpellChecker extends SolrSpellChecker {@Overridepublic SpellingResult getSuggestions(SpellingOptions options) throws IOException {- log.debug("getSuggestions: {}", options.tokens);-SpellingResult result = new SpellingResult();float accuracy =(options.accuracy == Float.MIN_VALUE) ? checker.getAccuracy() : options.accuracy;- for (Token token : options.tokens) {- if (token.length() == 0) {+ for (SpellCheckToken token : options.tokens) {+ String tokenText = token.text();+ if (tokenText.isEmpty()) {result.add(token, List.of());continue;}- String tokenText = token.toString();Term term = new Term(field, tokenText);int freq = options.reader.docFreq(term);int count =diff --git a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.javaindex 821df88c4c9..2af1f069d0e 100644--- a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java+++ b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java@@ -55,13 +55,14 @@ public class PossibilityIterator implements Iterator<PossibilityIterator.RankedS* Possible Correction".*/public PossibilityIterator(- Map<Token, LinkedHashMap<String, Integer>> suggestions,+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions,int maximumRequiredSuggestions,int maxEvaluations,boolean overlap) {this.suggestionsMayOverlap = overlap;- for (Map.Entry<Token, LinkedHashMap<String, Integer>> entry : suggestions.entrySet()) {- Token token = entry.getKey();+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> entry :+ suggestions.entrySet()) {+ SpellCheckToken token = entry.getKey();if (entry.getValue().size() == 0) {continue;}diff --git a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.javaindex 307e27471e9..55106c5c660 100644--- a/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java+++ b/solr/core/src/java/org/apache/solr/spelling/QueryConverter.java@@ -16,8 +16,8 @@*/package org.apache.solr.spelling;-import java.util.Collection;import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.TokenStream;import org.apache.solr.util.plugin.NamedListInitializedPlugin;/**@@ -57,10 +57,12 @@ public abstract class QueryConverter implements NamedListInitializedPlugin {public static final int TERM_IN_BOOLEAN_QUERY_FLAG = 131072;/**- * Returns the Collection of {@link Token}s for the query. Offsets on the Token should correspond- * to the correct offset in the origQuery+ * Returns a fresh {@link TokenStream} over the query's terms. Offsets should correspond to the+ * correct offset in the origQuery. The caller owns the returned stream's lifecycle: {@link+ * SpellCheckToken#drain(TokenStream)} resets it, reads it to the end and closes it, and the+ * spellcheckers read that list rather than this stream.*/- public abstract Collection<Token> convert(String original);+ public abstract TokenStream convert(String original);/** Set the analyzer to use. Must be set before any calls to convert. */public void setAnalyzer(Analyzer analyzer) {diff --git a/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.java b/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.javanew file mode 100644index 00000000000..92524dba6ec--- /dev/null+++ b/solr/core/src/java/org/apache/solr/spelling/QueryWordsTokenStream.java@@ -0,0 +1,149 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.spelling;++import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.List;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;+import org.apache.lucene.util.IOUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * Analyzes each parsed query word in turn and concatenates their analysis into a single stream,+ * shifting offsets to the original query string and setting {@link FlagsAttribute} from the+ * query-syntax parse ({@link QueryConverter#REQUIRED_TERM_FLAG} and friends). Each word is only+ * analyzed once this stream reaches it, not up front.+ */+final class QueryWordsTokenStream extends TokenStream {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ /** One query word as parsed from query syntax, prior to analysis. */+ record ParsedWord(String text, int startIndex, int flags) {}++ private final List<ParsedWord> words;+ private final Analyzer analyzer;+ private int nextWordIndex;+ private int currentWordIndex;+ private TokenStream current;+ private CharTermAttribute currentTermAtt;+ private OffsetAttribute currentOffsetAtt;+ private TypeAttribute currentTypeAtt;+ private PositionIncrementAttribute currentPosIncAtt;+ private PayloadAttribute currentPayloadAtt;++ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);+ private final PositionIncrementAttribute posIncAtt =+ addAttribute(PositionIncrementAttribute.class);+ private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);+ private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);++ QueryWordsTokenStream(List<ParsedWord> words, Analyzer analyzer) {+ this.words = words;+ this.analyzer = analyzer;+ }++ @Override+ public boolean incrementToken() throws IOException {+ while (true) {+ // A word whose analysis fails is skipped and the query's other words are still checked; an+ // IOException escaping here would fail the whole spellcheck request instead.+ if (current != null) {+ try {+ if (current.incrementToken()) {+ ParsedWord word = words.get(currentWordIndex);+ clearAttributes();+ termAtt.append(currentTermAtt);+ offsetAtt.setOffset(+ word.startIndex() + currentOffsetAtt.startOffset(),+ word.startIndex() + currentOffsetAtt.endOffset());+ typeAtt.setType(currentTypeAtt.type());+ posIncAtt.setPositionIncrement(currentPosIncAtt.getPositionIncrement());+ payloadAtt.setPayload(currentPayloadAtt.getPayload());+ flagsAtt.setFlags(word.flags());+ return true;+ }+ current.end();+ } catch (IOException e) {+ log.warn("Skipping the rest of query word '{}': its analysis failed", currentWord(), e);+ }+ IOUtils.closeWhileHandlingException(current);+ current = null;+ }+ if (nextWordIndex >= words.size()) {+ return false;+ }+ currentWordIndex = nextWordIndex++;+ try {+ current = analyzer.tokenStream("", currentWord());+ currentTermAtt = current.addAttribute(CharTermAttribute.class);+ currentOffsetAtt = current.addAttribute(OffsetAttribute.class);+ currentTypeAtt = current.addAttribute(TypeAttribute.class);+ currentPosIncAtt = current.addAttribute(PositionIncrementAttribute.class);+ currentPayloadAtt = current.addAttribute(PayloadAttribute.class);+ current.reset();+ } catch (IOException e) {+ log.warn("Skipping query word '{}': its analysis failed", currentWord(), e);+ IOUtils.closeWhileHandlingException(current);+ current = null;+ }+ }+ }++ private String currentWord() {+ return words.get(currentWordIndex).text();+ }++ @Override+ public void reset() throws IOException {+ super.reset();+ nextWordIndex = 0;+ currentWordIndex = 0;+ if (current != null) {+ current.close();+ current = null;+ }+ }++ @Override+ public void end() throws IOException {+ super.end();+ if (current != null) {+ current.end();+ }+ }++ @Override+ public void close() throws IOException {+ if (current != null) {+ current.close();+ current = null;+ }+ super.close();+ }+}diff --git a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.javaindex cadb3c76d6e..5f8ea9334ec 100644--- a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java+++ b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java@@ -19,11 +19,11 @@ package org.apache.solr.spelling;import java.util.Objects;public class ResultEntry {- public Token token;+ public SpellCheckToken token;public String suggestion;public int freq;- ResultEntry(Token t, String s, int f) {+ ResultEntry(SpellCheckToken t, String s, int f) {token = t;suggestion = s;freq = f;diff --git a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.javaindex f5a9989d9da..da0542243f6 100644--- a/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java+++ b/solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java@@ -129,7 +129,8 @@ public abstract class SolrSpellChecker {// create tokenSpellCheckResponse.Suggestion suggestion = mergeData.origVsSuggestion.get(original);- Token token = new Token(original, suggestion.getStartOffset(), suggestion.getEndOffset());+ SpellCheckToken token =+ new SpellCheckToken(original, suggestion.getStartOffset(), suggestion.getEndOffset());// get top 'count' suggestions out of 'sugQueue.size()' candidatesSuggestWord[] suggestions = new SuggestWord[Math.min(count, sugQueue.size())];diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.javaindex 344088c83ca..9faaa87a2d0 100644--- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java@@ -211,10 +211,10 @@ public class SpellCheckCollator {String corr = "";for (int i = 0; i < corrections.size(); i++) {SpellCheckCorrection correction = corrections.get(i);- Token tok = correction.getOriginal();+ SpellCheckToken tok = correction.getOriginal();// we are replacing the query in order, but injected terms might cause// illegal offsets due to previous replacements.- if (tok.getPositionIncrement() == 0) continue;+ if (tok.positionIncrement() == 0) continue;corr = correction.getCorrection();boolean addParenthesis = false;Character requiredOrProhibited = null;@@ -233,7 +233,7 @@ public class SpellCheckCollator {requiredOrProhibited = previousChar;}bump++;- } else if ((tok.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)+ } else if ((tok.flags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)== QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) {addParenthesis = true;corrSb.insert(indexOfSpace + bump, "AND ");diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.javaindex c040aca9b66..b8d465b5cbc 100644--- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCorrection.java@@ -17,12 +17,12 @@package org.apache.solr.spelling;public class SpellCheckCorrection {- private Token original;+ private SpellCheckToken original;private String originalAsString = null;private String correction;private int numberOfOccurences;- public Token getOriginal() {+ public SpellCheckToken getOriginal() {return original;}@@ -33,7 +33,7 @@ public class SpellCheckCorrection {return originalAsString;}- public void setOriginal(Token original) {+ public void setOriginal(SpellCheckToken original) {this.original = original;this.originalAsString = null;}diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.javanew file mode 100644index 00000000000..34f89a92768--- /dev/null+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckToken.java@@ -0,0 +1,119 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.spelling;++import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;+import org.apache.lucene.util.BytesRef;++/**+ * One term occurrence carried through the spellchecker API, and the key type of {@link+ * SpellingResult}. Unlike the old {@code Token} it replaces, this is a plain, immutable record --+ * not a Lucene {@code AttributeImpl} subclass.+ *+ * <p>A value type is needed here, rather than reading terms off a stream, because {@link+ * SolrSpellChecker#mergeSuggestions} keys suggestions by (text, offset) pairs deserialized from a+ * remote shard's response, where there is no {@link org.apache.lucene.analysis.TokenStream} to read+ * from at all.+ */+public record SpellCheckToken(+ String text,+ int startOffset,+ int endOffset,+ String type,+ int positionIncrement,+ int flags,+ BytesRef payload) {++ public SpellCheckToken {+ // PayloadAttribute hands out a buffer it refills for the next token, so a reference to it is+ // not a value. This record is a map key in SpellingResult and in ConjunctionSolrSpellChecker,+ // which need one whose bytes cannot change after it is stored.+ payload = payload == null ? null : BytesRef.deepCopyOf(payload);+ }++ public SpellCheckToken(String text, int startOffset, int endOffset) {+ this(text, startOffset, endOffset, "word", 1, 0, null);+ }++ /** The length of {@link #text()}, which the {@code Token} this replaced also offered. */+ public int length() {+ return text.length();+ }++ @Override+ public String toString() {+ return text;+ }++ /**+ * Reads {@code stream} to its end into a list, and closes it. This is the single point where a+ * {@link TokenStream} becomes values: every consumer of the spellcheck API reads every token, so+ * the stream's lifecycle need not reach any of them.+ */+ public static List<SpellCheckToken> drain(TokenStream stream) throws IOException {+ List<SpellCheckToken> tokens = new ArrayList<>();+ try (stream) {+ AttributeReader attrs = new AttributeReader(stream);+ stream.reset();+ while (stream.incrementToken()) {+ tokens.add(attrs.current());+ }+ stream.end();+ }+ return tokens;+ }++ /** Registers the six attributes {@link #drain} reads, once for the whole stream. */+ private static final class AttributeReader {+ private final CharTermAttribute termAtt;+ private final OffsetAttribute offsetAtt;+ private final TypeAttribute typeAtt;+ private final PositionIncrementAttribute posIncAtt;+ private final FlagsAttribute flagsAtt;+ private final PayloadAttribute payloadAtt;++ AttributeReader(TokenStream stream) {+ termAtt = stream.addAttribute(CharTermAttribute.class);+ offsetAtt = stream.addAttribute(OffsetAttribute.class);+ typeAtt = stream.addAttribute(TypeAttribute.class);+ posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);+ flagsAtt = stream.addAttribute(FlagsAttribute.class);+ payloadAtt = stream.addAttribute(PayloadAttribute.class);+ }++ /** Builds a {@link SpellCheckToken} from the stream's current position. */+ SpellCheckToken current() {+ return new SpellCheckToken(+ termAtt.toString(),+ offsetAtt.startOffset(),+ offsetAtt.endOffset(),+ typeAtt.type(),+ posIncAtt.getPositionIncrement(),+ flagsAtt.getFlags(),+ payloadAtt.getPayload());+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.javaindex 4975c1538ee..9f89c97a7f7 100644--- a/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingOptions.java@@ -16,7 +16,7 @@*/package org.apache.solr.spelling;-import java.util.Collection;+import java.util.List;import org.apache.lucene.index.IndexReader;import org.apache.lucene.search.spell.SuggestMode;import org.apache.solr.common.params.SolrParams;@@ -24,8 +24,11 @@ import org.apache.solr.common.params.SolrParams;/** */public class SpellingOptions {- /** The tokens to spell check */- public Collection<Token> tokens;+ /**+ * The terms to spell check, analyzed once. Several {@link SolrSpellChecker}s may read them --+ * e.g. via {@link ConjunctionSolrSpellChecker} -- and each reads this same list.+ */+ public List<SpellCheckToken> tokens;/** An optional {@link org.apache.lucene.index.IndexReader} */public IndexReader reader;@@ -52,24 +55,24 @@ public class SpellingOptions {public SpellingOptions() {}// A couple of convenience ones- public SpellingOptions(Collection<Token> tokens, int count) {+ public SpellingOptions(List<SpellCheckToken> tokens, int count) {this.tokens = tokens;this.count = count;}- public SpellingOptions(Collection<Token> tokens, IndexReader reader) {+ public SpellingOptions(List<SpellCheckToken> tokens, IndexReader reader) {this.tokens = tokens;this.reader = reader;}- public SpellingOptions(Collection<Token> tokens, IndexReader reader, int count) {+ public SpellingOptions(List<SpellCheckToken> tokens, IndexReader reader, int count) {this.tokens = tokens;this.reader = reader;this.count = count;}public SpellingOptions(- Collection<Token> tokens,+ List<SpellCheckToken> tokens,IndexReader reader,int count,SuggestMode suggestMode,@@ -86,7 +89,7 @@ public class SpellingOptions {}public SpellingOptions(- Collection<Token> tokens,+ List<SpellCheckToken> tokens,IndexReader reader,int count,int alternativeTermCount,diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.javaindex 6870459a377..9fda927588e 100644--- a/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingQueryConverter.java@@ -16,31 +16,24 @@*/package org.apache.solr.spelling;-import java.io.IOException;import java.util.ArrayList;-import java.util.Collection;import java.util.List;import java.util.regex.Matcher;import java.util.regex.Pattern;import org.apache.lucene.analysis.TokenStream;-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;/**- * Converts the query string to a Collection of Lucene tokens using a regular expression. Boolean- * operators AND, OR, NOT are skipped.+ * Converts the query string to a TokenStream using a regular expression. Boolean operators AND, OR,+ * NOT are skipped.** <p>Each term is checked to determine if it is optional, required or prohibited. Required terms- * output a {@link Token} with the {@link QueryConverter#REQUIRED_TERM_FLAG} set. Prohibited terms- * output a {@link Token} with the {@link QueryConverter#PROHIBITED_TERM_FLAG} set. If the query- * uses the plus (+) and minus (-) to denote required and prohibited, this determination will be- * accurate. In the case boolean AND/OR/NOTs are used, this converter makes an uninformed guess as- * to whether the term would likely behave as if it is Required or Prohibited and sets the flags- * accordingly. These flags are used downstream to generate collations for {@link- * WordBreakSolrSpellChecker}, in cases where an original term is split up into multiple Tokens.+ * output a token with the {@link QueryConverter#REQUIRED_TERM_FLAG} set. Prohibited terms output a+ * token with the {@link QueryConverter#PROHIBITED_TERM_FLAG} set. If the query uses the plus (+)+ * and minus (-) to denote required and prohibited, this determination will be accurate. In the case+ * boolean AND/OR/NOTs are used, this converter makes an uninformed guess as to whether the term+ * would likely behave as if it is Required or Prohibited and sets the flags accordingly. These+ * flags are used downstream to generate collations for {@link WordBreakSolrSpellChecker}, in cases+ * where an original term is split up into multiple tokens.** @since solr 1.3*/@@ -95,20 +88,22 @@ public class SpellingQueryConverter extends QueryConverter {protected Pattern QUERY_REGEX = Pattern.compile(PATTERN);/**- * Converts the original query string to a collection of Lucene Tokens.+ * Parses the original query string into a {@link TokenStream}; each matched query word is+ * analyzed lazily as the stream is consumed.** @param original the original query string- * @return a Collection of Lucene Tokens+ * @return a TokenStream over the query's terms, with {@code FlagsAttribute} set per word from the+ * query-syntax parse below*/@Override- public Collection<Token> convert(String original) {+ public TokenStream convert(String original) {if (original == null) { // this can happen with q.alt = and no query- return List.of();+ return new QueryWordsTokenStream(List.of(), analyzer);}boolean mightContainRangeQuery =(original.indexOf('[') != -1 || original.indexOf('{') != -1)&& (original.indexOf(']') != -1 || original.indexOf('}') != -1);- Collection<Token> result = new ArrayList<>();+ List<QueryWordsTokenStream.ParsedWord> words = new ArrayList<>();Matcher matcher = QUERY_REGEX.matcher(original);String nextWord = null;int nextStartIndex = 0;@@ -161,42 +156,17 @@ public class SpellingQueryConverter extends QueryConverter {&& ("NOT".equals(nextWord))) {flagValue = TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;}- try {- analyze(result, word, startIndex, flagValue);- } catch (IOException e) {- // TODO: shouldn't we log something?- }+ words.add(new QueryWordsTokenStream.ParsedWord(word, startIndex, flagValue));}if (lastBooleanOp != null) {- for (Token t : result) {- int f = t.getFlags();- t.setFlags(f |= QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG);+ for (int i = 0; i < words.size(); i++) {+ QueryWordsTokenStream.ParsedWord w = words.get(i);+ words.set(+ i,+ new QueryWordsTokenStream.ParsedWord(+ w.text(), w.startIndex(), w.flags() | QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG));}}- return result;- }-- protected void analyze(Collection<Token> result, String text, int offset, int flagsAttValue)- throws IOException {- TokenStream stream = analyzer.tokenStream("", text);- // TODO: support custom attributes- CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);- TypeAttribute typeAtt = stream.addAttribute(TypeAttribute.class);- PayloadAttribute payloadAtt = stream.addAttribute(PayloadAttribute.class);- PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);- OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);- stream.reset();- while (stream.incrementToken()) {- Token token = new Token();- token.copyBuffer(termAtt.buffer(), 0, termAtt.length());- token.setOffset(offset + offsetAtt.startOffset(), offset + offsetAtt.endOffset());- token.setFlags(flagsAttValue); // overwriting any flags already set...- token.setType(typeAtt.type());- token.setPayload(payloadAtt.getPayload());- token.setPositionIncrement(posIncAtt.getPositionIncrement());- result.add(token);- }- stream.end();- stream.close();+ return new QueryWordsTokenStream(words, analyzer);}}diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.javaindex 8cf29aa148f..fdec078404a 100644--- a/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java+++ b/solr/core/src/java/org/apache/solr/spelling/SpellingResult.java@@ -16,7 +16,6 @@*/package org.apache.solr.spelling;-import java.util.Collection;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -28,30 +27,25 @@ import java.util.Map;* @since solr 1.3*/public class SpellingResult {- private Collection<Token> tokens;/*** Key == token Value = Map -> key is the suggestion, value is the frequency of the token in the* collection*/- private Map<Token, LinkedHashMap<String, Integer>> suggestions = new LinkedHashMap<>();+ private Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions = new LinkedHashMap<>();- private Map<Token, Integer> tokenFrequency;+ private Map<SpellCheckToken, Integer> tokenFrequency;public static final int NO_FREQUENCY_INFO = -1;public SpellingResult() {}- public SpellingResult(Collection<Token> tokens) {- this.tokens = tokens;- }-/*** Adds a whole bunch of suggestions, and does not worry about frequency.** @param token The token to associate the suggestions with* @param suggestions The suggestions*/- public void add(Token token, List<String> suggestions) {+ public void add(SpellCheckToken token, List<String> suggestions) {LinkedHashMap<String, Integer> map = this.suggestions.get(token);if (map == null) {map = new LinkedHashMap<>();@@ -68,7 +62,7 @@ public class SpellingResult {* @param token original token* @param docFreq original token's document frequency*/- public void addFrequency(Token token, int docFreq) {+ public void addFrequency(SpellCheckToken token, int docFreq) {if (tokenFrequency == null) {tokenFrequency = new LinkedHashMap<>();}@@ -78,11 +72,11 @@ public class SpellingResult {/*** Suggestions must be added with the best suggestion first. ORDER is important.*- * @param token The {@link Token}- * @param suggestion The suggestion for the Token+ * @param token The {@link SpellCheckToken}+ * @param suggestion The suggestion for the token* @param docFreq The document frequency*/- public void add(Token token, String suggestion, int docFreq) {+ public void add(SpellCheckToken token, String suggestion, int docFreq) {LinkedHashMap<String, Integer> map = this.suggestions.get(token);// Don't bother adding if we already have this tokenif (map == null) {@@ -95,13 +89,13 @@ public class SpellingResult {/*** Gets the suggestions for the given token.*- * @param token The {@link Token} to look up+ * @param token The {@link SpellCheckToken} to look up* @return A LinkedHashMap of the suggestions. Key is the suggestion, value is the token frequency* in the index, else {@link #NO_FREQUENCY_INFO}.* <p>The suggestions are added in sorted order (i.e. best suggestion first) then the iterator* will return the suggestions in order*/- public LinkedHashMap<String, Integer> get(Token token) {+ public LinkedHashMap<String, Integer> get(SpellCheckToken token) {return suggestions.get(token);}@@ -111,7 +105,7 @@ public class SpellingResult {* @param token The token* @return The frequency or null*/- public Integer getTokenFrequency(Token token) {+ public Integer getTokenFrequency(SpellCheckToken token) {return tokenFrequency.get(token);}@@ -122,26 +116,15 @@ public class SpellingResult {/*** All the suggestions. The ordering of the inner LinkedHashMap is by best suggestion first.*- * @return The Map of suggestions for each Token. Key is the token, value is a LinkedHashMap whose+ * @return The Map of suggestions for each token. Key is the token, value is a LinkedHashMap whose* key is the Suggestion and the value is the frequency or {@link #NO_FREQUENCY_INFO} if* frequency info is not available.*/- public Map<Token, LinkedHashMap<String, Integer>> getSuggestions() {+ public Map<SpellCheckToken, LinkedHashMap<String, Integer>> getSuggestions() {return suggestions;}- public Map<Token, Integer> getTokenFrequency() {+ public Map<SpellCheckToken, Integer> getTokenFrequency() {return tokenFrequency;}-- /**- * @return The original tokens- */- public Collection<Token> getTokens() {- return tokens;- }-- public void setTokens(Collection<Token> tokens) {- this.tokens = tokens;- }}diff --git a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.javaindex da0bcbf3a55..84f4d5d7541 100644--- a/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java+++ b/solr/core/src/java/org/apache/solr/spelling/SuggestQueryConverter.java@@ -16,26 +16,18 @@*/package org.apache.solr.spelling;-import java.io.IOException;-import java.util.ArrayList;-import java.util.Collection;import java.util.List;+import org.apache.lucene.analysis.TokenStream;/** Passes the entire query string to the configured analyzer as-is. */public class SuggestQueryConverter extends SpellingQueryConverter {@Override- public Collection<Token> convert(String original) {+ public TokenStream convert(String original) {if (original == null) { // this can happen with q.alt = and no query- return List.of();+ return new QueryWordsTokenStream(List.of(), analyzer);}-- Collection<Token> result = new ArrayList<>();- try {- analyze(result, original, 0, 0);- } catch (IOException e) {- throw new RuntimeException(e);- }- return result;+ return new QueryWordsTokenStream(+ List.of(new QueryWordsTokenStream.ParsedWord(original, 0, 0)), analyzer);}}diff --git a/solr/core/src/java/org/apache/solr/spelling/Token.java b/solr/core/src/java/org/apache/solr/spelling/Token.javadeleted file mode 100644index c163784a8bd..00000000000--- a/solr/core/src/java/org/apache/solr/spelling/Token.java+++ /dev/null@@ -1,172 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.spelling;--import java.util.Objects;-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;-import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;-import org.apache.lucene.util.AttributeImpl;-import org.apache.lucene.util.AttributeReflector;-import org.apache.lucene.util.BytesRef;--/**- * A Token is an occurrence of a term from the text of a field. It consists of a term's text, the- * start and end offset of the term in the text of the field, and a type string.- *- * <p>The start and end offsets permit applications to re-associate a token with its source text,- * e.g., to display highlighted query terms in a document browser, or to show matching text- * fragments in a <a href="http://en.wikipedia.org/wiki/Key_Word_in_Context">KWIC</a> display, etc.- *- * <p>The type is a string, assigned by a lexical analyzer (a.k.a. tokenizer), naming the lexical or- * syntactic class that the token belongs to. For example an end of sentence marker token might be- * implemented with type "eos". The default token type is "word".- *- * <p>A Token can optionally have metadata (a.k.a. payload) in the form of a variable length byte- * array. Use {@link org.apache.lucene.index.PostingsEnum#getPayload()} to retrieve the payloads- * from the index.- *- * <p>A few things to note:- *- * <ul>- * <li>clear() initializes all of the fields to default values. This was changed in contrast to- * Lucene 2.4, but should affect no one.- * <li>Because <code>TokenStreams</code> can be chained, one cannot assume that the <code>Token's- * </code> current type is correct.- * <li>The startOffset and endOffset represent the start and offset in the source text, so be- * careful in adjusting them.- * <li>When caching a reusable token, clone it. When injecting a cached token into a stream that- * can be reset, clone it again.- * </ul>- */-@Deprecated(since = "7.0")-public class Token extends PackedTokenAttributeImpl implements FlagsAttribute, PayloadAttribute {-- // TODO Refactor the spellchecker API to use TokenStreams properly, rather than this hack-- private int flags;- private BytesRef payload;-- /** Constructs a Token will null text. */- public Token() {}-- /**- * Constructs a Token with the given term text, start and end offsets. The type defaults to- * "word." <b>NOTE:</b> for better indexing speed you should instead use the char[] termBuffer- * methods to set the term text.- *- * @param text term text- * @param start start offset in the source text- * @param end end offset in the source text- */- public Token(CharSequence text, int start, int end) {- append(text);- setOffset(start, end);- }-- /**- * {@inheritDoc}- *- * @see FlagsAttribute- */- @Override- public int getFlags() {- return flags;- }-- /**- * {@inheritDoc}- *- * @see FlagsAttribute- */- @Override- public void setFlags(int flags) {- this.flags = flags;- }-- /**- * {@inheritDoc}- *- * @see PayloadAttribute- */- @Override- public BytesRef getPayload() {- return this.payload;- }-- /**- * {@inheritDoc}- *- * @see PayloadAttribute- */- @Override- public void setPayload(BytesRef payload) {- this.payload = payload;- }-- /**- * Resets the term text, payload, flags, positionIncrement, positionLength, startOffset, endOffset- * and token type to default.- */- @Override- public void clear() {- super.clear();- flags = 0;- payload = null;- }-- @Override- public boolean equals(Object obj) {- if (obj == this) return true;- if (!(obj instanceof Token other)) return false;-- return (flags == other.flags && (Objects.equals(payload, other.payload)) && super.equals(obj));- }-- @Override- public int hashCode() {- int code = super.hashCode();- code = code * 31 + flags;- if (payload != null) {- code = code * 31 + payload.hashCode();- }- return code;- }-- @Override- public Token clone() {- final Token t = (Token) super.clone();- if (payload != null) {- t.payload = BytesRef.deepCopyOf(payload);- }- return t;- }-- @Override- public void copyTo(AttributeImpl target) {- super.copyTo(target);- ((FlagsAttribute) target).setFlags(flags);- ((PayloadAttribute) target).setPayload((payload == null) ? null : BytesRef.deepCopyOf(payload));- }-- @Override- public void reflectWith(AttributeReflector reflector) {- super.reflectWith(reflector);- reflector.reflect(FlagsAttribute.class, "flags", flags);- reflector.reflect(PayloadAttribute.class, "payload", payload);- }-}diff --git a/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.javaindex 1095acc2bd4..8f573054e74 100644--- a/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java+++ b/solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java@@ -158,9 +158,11 @@ public class WordBreakSolrSpellChecker extends SolrSpellChecker {int numSuggestions = options.count;StringBuilder sb = new StringBuilder();- Token[] tokenArr = options.tokens.toArray(new Token[0]);- List<Token> tokenArrWithSeparators = new ArrayList<>(options.tokens.size() + 2);- List<Term> termArr = new ArrayList<>(options.tokens.size() + 2);+ // an array because, unlike the other SolrSpellCheckers, this one reads several positions at+ // once to combine adjacent terms into one corrected phrase+ SpellCheckToken[] tokenArr = options.tokens.toArray(new SpellCheckToken[0]);+ List<SpellCheckToken> tokenArrWithSeparators = new ArrayList<>(tokenArr.length + 2);+ List<Term> termArr = new ArrayList<>(tokenArr.length + 2);List<ResultEntry> breakSuggestionList = new ArrayList<>();List<ResultEntry> noBreakSuggestionList = new ArrayList<>();boolean lastOneProhibited = false;@@ -168,13 +170,13 @@ public class WordBreakSolrSpellChecker extends SolrSpellChecker {boolean lastOneprocedesNewBooleanOp = false;for (int i = 0; i < tokenArr.length; i++) {boolean prohibited =- (tokenArr[i].getFlags() & QueryConverter.PROHIBITED_TERM_FLAG)+ (tokenArr[i].flags() & QueryConverter.PROHIBITED_TERM_FLAG)== QueryConverter.PROHIBITED_TERM_FLAG;boolean required =- (tokenArr[i].getFlags() & QueryConverter.REQUIRED_TERM_FLAG)+ (tokenArr[i].flags() & QueryConverter.REQUIRED_TERM_FLAG)== QueryConverter.REQUIRED_TERM_FLAG;boolean procedesNewBooleanOp =- (tokenArr[i].getFlags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)+ (tokenArr[i].flags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)== QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;if (i > 0&& (prohibited != lastOneProhibited@@ -234,8 +236,8 @@ public class WordBreakSolrSpellChecker extends SolrSpellChecker {}sb.append(tokenArrWithSeparators.get(i).toString());}- Token token =- new Token(+ SpellCheckToken token =+ new SpellCheckToken(sb.toString(),tokenArrWithSeparators.get(firstTermIndex).startOffset(),tokenArrWithSeparators.get(lastTermIndex).endOffset());@@ -316,7 +318,7 @@ public class WordBreakSolrSpellChecker extends SolrSpellChecker {private void addToResult(SpellingResult result,- Token token,+ SpellCheckToken token,int tokenFrequency,String suggestion,int suggestionFrequency) {@@ -329,7 +331,7 @@ public class WordBreakSolrSpellChecker extends SolrSpellChecker {}}- private int getCombineFrequency(IndexReader ir, Token token) throws IOException {+ private int getCombineFrequency(IndexReader ir, SpellCheckToken token) throws IOException {String[] words = spacePattern.split(token.toString());int result = 0;if (sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) {diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.javaindex c2a5d374ba7..21ce05aed6b 100644--- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java+++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java@@ -35,15 +35,14 @@ import org.apache.lucene.search.suggest.Lookup;import org.apache.lucene.search.suggest.Lookup.LookupResult;import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester;import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup;-import org.apache.lucene.util.CharsRef;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CloseHook;import org.apache.solr.core.SolrCore;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.spelling.SolrSpellChecker;+import org.apache.solr.spelling.SpellCheckToken;import org.apache.solr.spelling.SpellingOptions;import org.apache.solr.spelling.SpellingResult;-import org.apache.solr.spelling.Token;import org.apache.solr.spelling.suggest.fst.FSTLookupFactory;import org.apache.solr.spelling.suggest.tst.TSTLookupFactory;import org.slf4j.Logger;@@ -195,22 +194,17 @@ public class Suggester extends SolrSpellChecker {@Overridepublic SpellingResult getSuggestions(SpellingOptions options) throws IOException {- log.debug("getSuggestions: {}", options.tokens);if (lookup == null) {log.info("Lookup is null - invoke spellchecker.build first");return EMPTY_RESULT;}SpellingResult res = new SpellingResult();- CharsRef scratch = new CharsRef();- for (Token t : options.tokens) {- scratch.chars = t.buffer();- scratch.offset = 0;- scratch.length = t.length();+ for (SpellCheckToken t : options.tokens) {boolean onlyMorePopular =(options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR)&& !(lookup instanceof WFSTCompletionLookup)&& !(lookup instanceof AnalyzingSuggester);- List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular, options.count);+ List<LookupResult> suggestions = lookup.lookup(t.text(), onlyMorePopular, options.count);if (suggestions == null) {continue;}diff --git a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.javaindex 3a642df514f..0b17667ed01 100644--- a/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/DirectSolrSpellCheckerTest.java@@ -16,9 +16,10 @@*/package org.apache.solr.spelling;-import java.util.Collection;import java.util.List;import java.util.Map;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.core.KeywordAnalyzer;import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.params.SpellingParams;@@ -51,6 +52,11 @@ public class DirectSolrSpellCheckerTest extends SolrTestCaseJ4 {queryConverter.init(new NamedList<>());}+ /** A stream that emits exactly one token whose term text is empty. */+ private static TokenStream singleEmptyTermTokenStream() {+ return new KeywordAnalyzer().tokenStream("", "");+ }+@Testpublic void test() throws Exception {DirectSolrSpellChecker checker = new DirectSolrSpellChecker();@@ -67,11 +73,11 @@ public class DirectSolrSpellCheckerTest extends SolrTestCaseJ4 {searcher -> {// check that 'fob' is corrected to 'foo'- Collection<Token> tokens = queryConverter.convert("fob");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(queryConverter.convert("fob"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.getIndexReader());SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertFalse("suggestions shouldn't be empty", suggestions.isEmpty());Map.Entry<String, Integer> entry = suggestions.entrySet().iterator().next();assertEquals("foo", entry.getKey());@@ -81,18 +87,18 @@ public class DirectSolrSpellCheckerTest extends SolrTestCaseJ4 {(int) entry.getValue());// check that 'super' is *not* corrected- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());// Check empty token due to spellcheck.q = ""- spellOpts.tokens = List.of(new Token("", 0, 0));+ spellOpts.tokens = SpellCheckToken.drain(singleEmptyTermTokenStream());result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(new SpellCheckToken("", 0, 0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());return null;@@ -143,11 +149,12 @@ public class DirectSolrSpellCheckerTest extends SolrTestCaseJ4 {h.getCore().withSearcher(searcher -> {- Collection<Token> tokens = queryConverter.convert("anothar");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(queryConverter.convert("anothar"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.getIndexReader());SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("suggestions shouldn't be null", suggestions);if (limitQueryLength) {diff --git a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.javaindex 5065e0c398a..5c90814ff9a 100644--- a/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/FileBasedSpellCheckerTest.java@@ -18,9 +18,10 @@ package org.apache.solr.spelling;import java.nio.file.Files;import java.nio.file.Path;-import java.util.Collection;import java.util.List;import java.util.Map;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.core.KeywordAnalyzer;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;import org.apache.solr.SolrTestCaseJ4;@@ -59,6 +60,11 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {queryConverter = null;}+ /** A stream that emits exactly one token whose term text is empty. */+ private static TokenStream singleEmptyTermTokenStream() {+ return new KeywordAnalyzer().tokenStream("", "");+ }+@Testpublic void test() throws Exception {FileBasedSpellChecker checker = new FileBasedSpellChecker();@@ -79,11 +85,11 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {h.getCore().withSearcher(searcher -> {- Collection<Token> tokens = queryConverter.convert("fob");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(queryConverter.convert("fob"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.getIndexReader());SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));Map.Entry<String, Integer> entry = suggestions.entrySet().iterator().next();assertEquals(entry.getKey() + " is not equal to " + "foo", "foo", entry.getKey());assertEquals(@@ -91,18 +97,18 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {SpellingResult.NO_FREQUENCY_INFO,(int) entry.getValue());- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());// Check empty token due to spellcheck.q = ""- spellOpts.tokens = List.of(new Token("", 0, 0));+ spellOpts.tokens = SpellCheckToken.drain(singleEmptyTermTokenStream());result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(new SpellCheckToken("", 0, 0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());return null;@@ -127,7 +133,7 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {assertEquals(dictName + " is not equal to " + "external", "external", dictName);checker.build(core, null);- Collection<Token> tokens = queryConverter.convert("Solar");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(queryConverter.convert("Solar"));h.getCore().withSearcher(searcher -> {@@ -135,7 +141,7 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);// should be lowercased, b/c we are using a lowercasing analyzer- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertEquals("suggestions Size: " + suggestions.size() + " is not: " + 1,1,@@ -148,10 +154,10 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {(int) entry.getValue());// test something not in the spell checker- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());return null;@@ -180,12 +186,12 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {h.getCore().withSearcher(searcher -> {- Collection<Token> tokens = queryConverter.convert("solar");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(queryConverter.convert("solar"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.getIndexReader());SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);// should be lowercased, b/c we are using a lowercasing analyzer- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertEquals("suggestions Size: " + suggestions.size() + " is not: " + 1,1,@@ -197,10 +203,10 @@ public class FileBasedSpellCheckerTest extends SolrTestCaseJ4 {SpellingResult.NO_FREQUENCY_INFO,(int) entry.getValue());- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result shouldn't be null", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("suggestions shouldn't be null", suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());return null;diff --git a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.javaindex 6ee537f814d..71bcf056be7 100644--- a/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java@@ -18,11 +18,12 @@ package org.apache.solr.spelling;import java.nio.file.Files;import java.nio.file.Path;-import java.util.Collection;import java.util.Comparator;import java.util.Date;import java.util.List;import java.util.Map;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.core.KeywordAnalyzer;import org.apache.lucene.analysis.core.WhitespaceAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -126,12 +127,13 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {checker.build(core, searcher);IndexReader reader = searcher.getIndexReader();- Collection<Token> tokens = queryConverter.convert("documemt");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(queryConverter.convert("documemt"));SpellingOptions spellOpts = new SpellingOptions(tokens, reader);SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);// should be lowercased, b/c we are using a lowercasing analyzer- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("documemt is null and it shouldn't be", suggestions);assertEquals("documemt Size: " + suggestions.size() + " is not: " + 1, 1, suggestions.size());@@ -144,32 +146,32 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {(int) entry.getValue());// test something not in the spell checker- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertEquals("suggestions size should be 0", 0, suggestions.size());// test something that is spelled correctly- spellOpts.tokens = queryConverter.convert("document");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("document"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNull("suggestions is null and it shouldn't be", suggestions);// Has multiple possibilities, but the exact exists, so that should be returned- spellOpts.tokens = queryConverter.convert("red");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("red"));spellOpts.count = 2;result = checker.getSuggestions(spellOpts);assertNotNull(result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNull("suggestions is not null and it should be", suggestions);// Try out something which should have multiple suggestions- spellOpts.tokens = queryConverter.convert("bug");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("bug"));result = checker.getSuggestions(spellOpts);assertNotNull(result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNotNull(suggestions);assertEquals("suggestions Size: " + suggestions.size() + " is not: " + 2,@@ -197,16 +199,21 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {(int) entry.getValue());// Check empty token due to spellcheck.q = ""- spellOpts.tokens = List.of(new Token("", 0, 0));+ spellOpts.tokens = SpellCheckToken.drain(singleEmptyTermTokenStream());result = checker.getSuggestions(spellOpts);assertNotNull(result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(new SpellCheckToken("", 0, 0));assertNotNull(suggestions);assertTrue("suggestions should be empty", suggestions.isEmpty());return null;});}+ /** A stream that emits exactly one token whose term text is empty. */+ private static TokenStream singleEmptyTermTokenStream() {+ return new KeywordAnalyzer().tokenStream("", "");+ }+@Testpublic void testExtendedResults() throws Exception {IndexBasedSpellChecker checker = new IndexBasedSpellChecker();@@ -230,14 +237,15 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {checker.build(core, searcher);IndexReader reader = searcher.getIndexReader();- Collection<Token> tokens = queryConverter.convert("documemt");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(queryConverter.convert("documemt"));SpellingOptions spellOpts =new SpellingOptions(tokens, reader, 1, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);// should be lowercased, b/c we are using a lowercasing analyzer- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("documemt is null and it shouldn't be", suggestions);assertEquals("documemt Size: " + suggestions.size() + " is not: " + 1, 1, suggestions.size());@@ -247,16 +255,16 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {assertEquals(entry.getValue() + " does not equal: " + 2, 2, (int) entry.getValue());// test something not in the spell checker- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertEquals("suggestions size should be 0", 0, suggestions.size());- spellOpts.tokens = queryConverter.convert("document");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("document"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNull("suggestions is not null and it should be", suggestions);return null;});@@ -350,14 +358,14 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {checker.build(core, searcher);IndexReader reader = searcher.getIndexReader();- Collection<Token> tokens = queryConverter.convert("flesh");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(queryConverter.convert("flesh"));SpellingOptions spellOpts =new SpellingOptions(tokens, reader, 1, SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX, true, 0.5f, null);SpellingResult result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);// should be lowercased, b/c we are using a lowercasing analyzer- Map<String, Integer> suggestions = result.get(spellOpts.tokens.iterator().next());+ Map<String, Integer> suggestions = result.get(spellOpts.tokens.get(0));assertNotNull("flesh is null and it shouldn't be", suggestions);assertEquals("flesh Size: " + suggestions.size() + " is not: " + 1, 1, suggestions.size());@@ -366,16 +374,16 @@ public class IndexBasedSpellCheckerTest extends SolrTestCaseJ4 {assertEquals(entry.getValue() + " does not equal: " + 1, 1, (int) entry.getValue());// test something not in the spell checker- spellOpts.tokens = queryConverter.convert("super");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("super"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertEquals("suggestions size should be 0", 0, suggestions.size());- spellOpts.tokens = queryConverter.convert("Caroline");+ spellOpts.tokens = SpellCheckToken.drain(queryConverter.convert("Caroline"));result = checker.getSuggestions(spellOpts);assertNotNull("result is null and it shouldn't be", result);- suggestions = result.get(spellOpts.tokens.iterator().next());+ suggestions = result.get(spellOpts.tokens.get(0));assertNull("suggestions is not null and it should be", suggestions);return null;});diff --git a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.javaindex 9cda54676b1..3342f6a5082 100644--- a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java+++ b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java@@ -16,53 +16,18 @@*/package org.apache.solr.spelling;-import java.io.IOException;-import java.util.Collection;-import java.util.HashSet;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.core.WhitespaceAnalyzer;-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;/*** @since solr 1.3*/class SimpleQueryConverter extends SpellingQueryConverter {- @Override- public Collection<Token> convert(String origQuery) {- Collection<Token> result = new HashSet<>();-- try (WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer();- TokenStream ts = analyzer.tokenStream("", origQuery)) {- // TODO: support custom attributes- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);- TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);- FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);- PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);- PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);+ private static final WhitespaceAnalyzer ANALYZER = new WhitespaceAnalyzer();- ts.reset();-- while (ts.incrementToken()) {- Token tok = new Token();- tok.copyBuffer(termAtt.buffer(), 0, termAtt.length());- tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());- tok.setFlags(flagsAtt.getFlags());- tok.setPayload(payloadAtt.getPayload());- tok.setPositionIncrement(posIncAtt.getPositionIncrement());- tok.setType(typeAtt.type());- result.add(tok);- }- ts.end();- return result;- } catch (IOException e) {- throw new RuntimeException(e);- }+ @Override+ public TokenStream convert(String origQuery) {+ return ANALYZER.tokenStream("", origQuery);}}diff --git a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.javaindex d402044b4ce..365744601c1 100644--- a/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/SpellPossibilityIteratorTest.java@@ -25,10 +25,10 @@ import org.junit.Before;import org.junit.Test;public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {- private static final Token TOKEN_AYE = new Token("AYE", 0, 3);- private static final Token TOKEN_BEE = new Token("BEE", 4, 7);- private static final Token TOKEN_AYE_BEE = new Token("AYE BEE", 0, 7);- private static final Token TOKEN_CEE = new Token("CEE", 8, 11);+ private static final SpellCheckToken TOKEN_AYE = new SpellCheckToken("AYE", 0, 3);+ private static final SpellCheckToken TOKEN_BEE = new SpellCheckToken("BEE", 4, 7);+ private static final SpellCheckToken TOKEN_AYE_BEE = new SpellCheckToken("AYE BEE", 0, 7);+ private static final SpellCheckToken TOKEN_CEE = new SpellCheckToken("CEE", 8, 11);private LinkedHashMap<String, Integer> AYE;private LinkedHashMap<String, Integer> BEE;@@ -87,26 +87,26 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {@Testpublic void testScalability() {- Map<Token, LinkedHashMap<String, Integer>> lotsaSuggestions = new LinkedHashMap<>();+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> lotsaSuggestions = new LinkedHashMap<>();lotsaSuggestions.put(TOKEN_AYE, AYE);lotsaSuggestions.put(TOKEN_BEE, BEE);lotsaSuggestions.put(TOKEN_CEE, CEE);- lotsaSuggestions.put(new Token("AYE1", 0, 3), AYE);- lotsaSuggestions.put(new Token("BEE1", 4, 7), BEE);- lotsaSuggestions.put(new Token("CEE1", 8, 11), CEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE1", 0, 3), AYE);+ lotsaSuggestions.put(new SpellCheckToken("BEE1", 4, 7), BEE);+ lotsaSuggestions.put(new SpellCheckToken("CEE1", 8, 11), CEE);- lotsaSuggestions.put(new Token("AYE2", 0, 3), AYE);- lotsaSuggestions.put(new Token("BEE2", 4, 7), BEE);- lotsaSuggestions.put(new Token("CEE2", 8, 11), CEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE2", 0, 3), AYE);+ lotsaSuggestions.put(new SpellCheckToken("BEE2", 4, 7), BEE);+ lotsaSuggestions.put(new SpellCheckToken("CEE2", 8, 11), CEE);- lotsaSuggestions.put(new Token("AYE3", 0, 3), AYE);- lotsaSuggestions.put(new Token("BEE3", 4, 7), BEE);- lotsaSuggestions.put(new Token("CEE3", 8, 11), CEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE3", 0, 3), AYE);+ lotsaSuggestions.put(new SpellCheckToken("BEE3", 4, 7), BEE);+ lotsaSuggestions.put(new SpellCheckToken("CEE3", 8, 11), CEE);- lotsaSuggestions.put(new Token("AYE4", 0, 3), AYE);- lotsaSuggestions.put(new Token("BEE4", 4, 7), BEE);- lotsaSuggestions.put(new Token("CEE4", 8, 11), CEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE4", 0, 3), AYE);+ lotsaSuggestions.put(new SpellCheckToken("BEE4", 4, 7), BEE);+ lotsaSuggestions.put(new SpellCheckToken("CEE4", 8, 11), CEE);PossibilityIterator iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000, false);int count = 0;@@ -116,10 +116,10 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {}assertEquals(1000, count);- lotsaSuggestions.put(new Token("AYE_BEE1", 0, 7), AYE_BEE);- lotsaSuggestions.put(new Token("AYE_BEE2", 0, 7), AYE_BEE);- lotsaSuggestions.put(new Token("AYE_BEE3", 0, 7), AYE_BEE);- lotsaSuggestions.put(new Token("AYE_BEE4", 0, 7), AYE_BEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE1", 0, 7), AYE_BEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE2", 0, 7), AYE_BEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE3", 0, 7), AYE_BEE);+ lotsaSuggestions.put(new SpellCheckToken("AYE_BEE4", 0, 7), AYE_BEE);iter = new PossibilityIterator(lotsaSuggestions, 1000, 10000, true);count = 0;while (iter.hasNext()) {@@ -131,7 +131,7 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {@Testpublic void testSpellPossibilityIterator() {- Map<Token, LinkedHashMap<String, Integer>> suggestions = new LinkedHashMap<>();+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> suggestions = new LinkedHashMap<>();suggestions.put(TOKEN_AYE, AYE);suggestions.put(TOKEN_BEE, BEE);suggestions.put(TOKEN_CEE, CEE);@@ -184,7 +184,8 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {@Testpublic void testOverlappingTokens() {- Map<Token, LinkedHashMap<String, Integer>> overlappingSuggestions = new LinkedHashMap<>();+ Map<SpellCheckToken, LinkedHashMap<String, Integer>> overlappingSuggestions =+ new LinkedHashMap<>();overlappingSuggestions.put(TOKEN_AYE, AYE);overlappingSuggestions.put(TOKEN_BEE, BEE);overlappingSuggestions.put(TOKEN_AYE_BEE, AYE_BEE);@@ -197,10 +198,10 @@ public class SpellPossibilityIteratorTest extends SolrTestCaseJ4 {Set<PossibilityIterator.RankedSpellPossibility> dupChecker = new HashSet<>();while (iter.hasNext()) {PossibilityIterator.RankedSpellPossibility rsp = iter.next();- Token a = null;- Token b = null;- Token ab = null;- Token c = null;+ SpellCheckToken a = null;+ SpellCheckToken b = null;+ SpellCheckToken ab = null;+ SpellCheckToken c = null;for (SpellCheckCorrection scc : rsp.corrections) {if (scc.getOriginal().equals(TOKEN_AYE)) {a = scc.getOriginal();diff --git a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.javaindex 42172c89e45..d1f16a0d38f 100644--- a/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java@@ -16,10 +16,15 @@*/package org.apache.solr.spelling;-import java.util.ArrayList;-import java.util.Collection;+import java.io.IOException;import java.util.List;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.TokenFilter;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.WhitespaceAnalyzer;+import org.apache.lucene.analysis.core.WhitespaceTokenizer;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.solr.SolrTestCase;import org.apache.solr.common.util.NamedList;import org.junit.Test;@@ -32,17 +37,17 @@ import org.junit.Test;public class SpellingQueryConverterTest extends SolrTestCase {@Test- public void test() {+ public void test() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());- Collection<Token> tokens = converter.convert("field:foo");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("field:foo"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not: " + 1, 1, tokens.size());}@Test- public void testNumeric() {+ public void testNumeric() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());@@ -58,7 +63,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {};int[] tokensToExpect = {1, 1, 2, 2, 2, 2, 2, 2};for (int i = 0; i < queries.length; i++) {- Collection<Token> tokens = converter.convert(queries[i]);+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert(queries[i]));assertEquals("tokens Size: " + tokens.size() + " is not: " + tokensToExpect[i],tokens.size(),@@ -67,24 +72,24 @@ public class SpellingQueryConverterTest extends SolrTestCase {}@Test- public void testSpecialChars() {+ public void testSpecialChars() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());String original = "field_with_underscore:value_with_underscore";- Collection<Token> tokens = converter.convert(original);+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert(original));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));original = "field_with_digits123:value_with_digits123";- tokens = converter.convert(original);+ tokens = SpellCheckToken.drain(converter.convert(original));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));original = "field-with-hyphens:value-with-hyphens";- tokens = converter.convert(original);+ tokens = SpellCheckToken.drain(converter.convert(original));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));@@ -97,7 +102,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {// assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));original = "foo:bar^5.0";- tokens = converter.convert(original);+ tokens = SpellCheckToken.drain(converter.convert(original));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));@@ -105,22 +110,16 @@ public class SpellingQueryConverterTest extends SolrTestCase {String firstKeyword = "value1";String secondKeyword = "value2";original = "field-with-parenthesis:(" + firstKeyword + " " + secondKeyword + ")";- tokens = converter.convert(original);+ tokens = SpellCheckToken.drain(converter.convert(original));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 2", 2, tokens.size());assertTrue("Token offsets do not match", isOffsetCorrect(original, tokens));- assertEquals(- "first Token is not " + firstKeyword,- new ArrayList<>(tokens).get(0).toString(),- firstKeyword);- assertEquals(- "second Token is not " + secondKeyword,- new ArrayList<>(tokens).get(1).toString(),- secondKeyword);+ assertEquals("first Token is not " + firstKeyword, tokens.get(0).toString(), firstKeyword);+ assertEquals("second Token is not " + secondKeyword, tokens.get(1).toString(), secondKeyword);}- private boolean isOffsetCorrect(String s, Collection<Token> tokens) {- for (Token token : tokens) {+ private boolean isOffsetCorrect(String s, List<SpellCheckToken> tokens) {+ for (SpellCheckToken token : tokens) {int start = token.startOffset();int end = token.endOffset();if (!s.substring(start, end).equals(token.toString())) return false;@@ -129,50 +128,52 @@ public class SpellingQueryConverterTest extends SolrTestCase {}@Test- public void testUnicode() {+ public void testUnicode() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());// chinese text value- Collection<Token> tokens = converter.convert("text_field:我购买了道具和服装。");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(converter.convert("text_field:我购买了道具和服装。"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());- tokens = converter.convert("text_购field:我购买了道具和服装。");+ tokens = SpellCheckToken.drain(converter.convert("text_购field:我购买了道具和服装。"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());- tokens = converter.convert("text_field:我购xyz买了道具和服装。");+ tokens = SpellCheckToken.drain(converter.convert("text_field:我购xyz买了道具和服装。"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 1", 1, tokens.size());}@Test- public void testMultipleClauses() {+ public void testMultipleClauses() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());// two field:value pairs should give two tokens- Collection<Token> tokens = converter.convert("买text_field:我购买了道具和服装。 field2:bar");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(converter.convert("买text_field:我购买了道具和服装。 field2:bar"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 2", 2, tokens.size());// a field:value pair and a search term should give two tokens- tokens = converter.convert("text_field:我购买了道具和服装。 bar");+ tokens = SpellCheckToken.drain(converter.convert("text_field:我购买了道具和服装。 bar"));assertNotNull("tokens is null and it shouldn't be", tokens);assertEquals("tokens Size: " + tokens.size() + " is not 2", 2, tokens.size());}@Test- public void testRequiredOrProhibitedFlags() {+ public void testRequiredOrProhibitedFlags() throws IOException {SpellingQueryConverter converter = new SpellingQueryConverter();converter.init(new NamedList<>());converter.setAnalyzer(new WhitespaceAnalyzer());{- List<Token> tokens = new ArrayList<>(converter.convert("aaa bbb ccc"));+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("aaa bbb ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 should be optional",@@ -185,7 +186,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasRequiredFlag(tokens.get(2)) && !hasProhibitedFlag(tokens.get(2)));}{- List<Token> tokens = new ArrayList<>(converter.convert("+aaa bbb -ccc"));+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("+aaa bbb -ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 should be required",@@ -198,7 +199,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasRequiredFlag(tokens.get(2)) && hasProhibitedFlag(tokens.get(2)));}{- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND bbb ccc"));+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("aaa AND bbb ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 doesn't precede n.b.o.",@@ -211,7 +212,7 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));}{- List<Token> tokens = new ArrayList<>(converter.convert("aaa OR bbb OR ccc"));+ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("aaa OR bbb OR ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 doesn't precede n.b.o.",@@ -224,7 +225,8 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));}{- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND bbb NOT ccc"));+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(converter.convert("aaa AND bbb NOT ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 doesn't precede n.b.o.",@@ -236,7 +238,8 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));}{- List<Token> tokens = new ArrayList<>(converter.convert("aaa NOT bbb AND ccc"));+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(converter.convert("aaa NOT bbb AND ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0)));@@ -247,7 +250,8 @@ public class SpellingQueryConverterTest extends SolrTestCase {!hasNBOFlag(tokens.get(2)) && hasInBooleanFlag(tokens.get(0)));}{- List<Token> tokens = new ArrayList<>(converter.convert("aaa AND NOT bbb AND ccc"));+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(converter.convert("aaa AND NOT bbb AND ccc"));assertTrue("Should have 3 tokens", tokens != null && tokens.size() == 3);assertTrue("token 1 precedes n.b.o.", hasNBOFlag(tokens.get(0)) && hasInBooleanFlag(tokens.get(0)));@@ -259,22 +263,64 @@ public class SpellingQueryConverterTest extends SolrTestCase {}}- private boolean hasRequiredFlag(Token t) {- return (t.getFlags() & QueryConverter.REQUIRED_TERM_FLAG) == QueryConverter.REQUIRED_TERM_FLAG;+ /**+ * A query word whose analysis throws is skipped, and the query's other words are still converted.+ */+ @Test+ public void testWordFailingAnalysisIsSkipped() throws IOException {+ SpellingQueryConverter converter = new SpellingQueryConverter();+ converter.init(new NamedList<>());+ converter.setAnalyzer(+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer source = new WhitespaceTokenizer();+ return new TokenStreamComponents(source, new FailOnTermFilter(source, "bbb"));+ }+ });++ List<SpellCheckToken> tokens = SpellCheckToken.drain(converter.convert("aaa bbb ccc"));++ assertEquals(List.of("aaa", "ccc"), tokens.stream().map(SpellCheckToken::text).toList());+ }++ /** Fails on one term, the way a filter reading an external dictionary would on a bad read. */+ private static class FailOnTermFilter extends TokenFilter {+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);+ private final String failOn;++ FailOnTermFilter(TokenStream input, String failOn) {+ super(input);+ this.failOn = failOn;+ }++ @Override+ public boolean incrementToken() throws IOException {+ if (!input.incrementToken()) {+ return false;+ }+ if (failOn.contentEquals(termAtt)) {+ throw new IOException("analysis of '" + failOn + "' failed");+ }+ return true;+ }+ }++ private boolean hasRequiredFlag(SpellCheckToken t) {+ return (t.flags() & QueryConverter.REQUIRED_TERM_FLAG) == QueryConverter.REQUIRED_TERM_FLAG;}- private boolean hasProhibitedFlag(Token t) {- return (t.getFlags() & QueryConverter.PROHIBITED_TERM_FLAG)- == QueryConverter.PROHIBITED_TERM_FLAG;+ private boolean hasProhibitedFlag(SpellCheckToken t) {+ return (t.flags() & QueryConverter.PROHIBITED_TERM_FLAG) == QueryConverter.PROHIBITED_TERM_FLAG;}- private boolean hasNBOFlag(Token t) {- return (t.getFlags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)+ private boolean hasNBOFlag(SpellCheckToken t) {+ return (t.flags() & QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG)== QueryConverter.TERM_PRECEDES_NEW_BOOLEAN_OPERATOR_FLAG;}- private boolean hasInBooleanFlag(Token t) {- return (t.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)+ private boolean hasInBooleanFlag(SpellCheckToken t) {+ return (t.flags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG)== QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG;}}diff --git a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.javaindex e664e408287..ad0729b123b 100644--- a/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java+++ b/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java@@ -16,7 +16,7 @@*/package org.apache.solr.spelling;-import java.util.Collection;+import java.io.IOException;import java.util.regex.Pattern;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.LowerCaseFilter;@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.KeywordTokenizer;import org.apache.lucene.analysis.miscellaneous.TrimFilter;import org.apache.lucene.analysis.pattern.PatternReplaceFilter;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.analysis.MockTokenizer;@@ -66,12 +67,18 @@ public class TestSuggestSpellingConverter extends BaseTokenStreamTestCase {}public void assertConvertsTo(String text, String expected[]) {- Collection<Token> tokens = converter.convert(text);- assertEquals(tokens.size(), expected.length);- int i = 0;- for (Token token : tokens) {- assertEquals(token.toString(), expected[i]);- i++;+ try (TokenStream stream = converter.convert(text)) {+ stream.reset();+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);+ int i = 0;+ while (stream.incrementToken()) {+ assertEquals(termAtt.toString(), expected[i]);+ i++;+ }+ stream.end();+ assertEquals(i, expected.length);+ } catch (IOException e) {+ throw new RuntimeException(e);}}}diff --git a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.javaindex 87db8f6176a..4922b3c5c1e 100644--- a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java+++ b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java@@ -16,8 +16,8 @@*/package org.apache.solr.spelling;-import java.util.Collection;import java.util.LinkedHashMap;+import java.util.List;import java.util.Map;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.util.LuceneTestCase.SuppressTempFileChecks;@@ -73,7 +73,7 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {{// Prior to SOLR-8175, the required term would cause an AIOOBE.- Collection<Token> tokens = qc.convert("+pine apple good ness");+ List<SpellCheckToken> tokens = SpellCheckToken.drain(qc.convert("+pine apple good ness"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.get().getIndexReader(), 10);SpellingResult result = checker.getSuggestions(spellOpts);searcher.decref();@@ -81,7 +81,8 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {assertEquals(5, result.getSuggestions().size());}- Collection<Token> tokens = qc.convert("paintable pine apple good ness");+ List<SpellCheckToken> tokens =+ SpellCheckToken.drain(qc.convert("paintable pine apple good ness"));SpellingOptions spellOpts = new SpellingOptions(tokens, searcher.get().getIndexReader(), 10);SpellingResult result = checker.getSuggestions(spellOpts);searcher.decref();@@ -89,8 +90,9 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {assertTrue(result != null && result.getSuggestions() != null);assertEquals(9, result.getSuggestions().size());- for (Map.Entry<Token, LinkedHashMap<String, Integer>> s : result.getSuggestions().entrySet()) {- Token orig = s.getKey();+ for (Map.Entry<SpellCheckToken, LinkedHashMap<String, Integer>> s :+ result.getSuggestions().entrySet()) {+ SpellCheckToken orig = s.getKey();String[] corr = s.getValue().keySet().toArray(new String[0]);if (orig.toString().equals("paintable")) {assertEquals(0, orig.startOffset());diff --git a/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java b/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.javaindex 4c36106fd42..a5e1a214be3 100644--- a/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java+++ b/solr/test-framework/src/java/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java@@ -24,9 +24,9 @@ import java.util.List;import org.apache.solr.core.SolrCore;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.spelling.SolrSpellChecker;+import org.apache.solr.spelling.SpellCheckToken;import org.apache.solr.spelling.SpellingOptions;import org.apache.solr.spelling.SpellingResult;-import org.apache.solr.spelling.Token;/** A Dummy SpellChecker for testing purposes */public class DummyCustomParamSpellChecker extends SolrSpellChecker {@@ -54,7 +54,7 @@ public class DummyCustomParamSpellChecker extends SolrSpellChecker {int i = 0;for (String name : lst) {String value = options.customParams.get(name);- result.add(new Token(name, i, i + 1), List.of(value));+ result.add(new SpellCheckToken(name, i, i + 1), List.of(value));i += 2;}return result;