Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API (#1557)

vinayak hegde502faf224ba2
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2ca207bbf42..26a88337aac 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -146,6 +146,10 @@ Optimizations
* SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh)
+* SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API. This removes the use of ThreadLocal for
+ stored field state, reducing heap usage especially for high-core-count, high-field-count, high-thread-count
+ cases (Vinayak Hegde, Christine Poerschke, Kevin Risden, David Smiley, Michael Gibney)
+
Bug Fixes
---------------------
* SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)
diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
index 30c310ca0e3..a612b27809a 100644
--- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
+++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
@@ -30,6 +30,7 @@ import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,13 +81,19 @@ public class QuerySenderListener extends AbstractSolrEventListener {
NamedList<?> values = rsp.getValues();
for (int i = 0; i < values.size(); i++) {
Object o = values.getVal(i);
+ SolrDocumentFetcher docFetcher = null;
if (o instanceof ResultContext) {
- o = ((ResultContext) o).getDocList();
+ ResultContext ctx = (ResultContext) o;
+ o = ctx.getDocList();
+ docFetcher = ctx.getDocFetcher();
}
if (o instanceof DocList) {
DocList docs = (DocList) o;
+ if (docFetcher == null) {
+ docFetcher = newSearcher.getDocFetcher();
+ }
for (DocIterator iter = docs.iterator(); iter.hasNext(); ) {
- newSearcher.doc(iter.nextDoc());
+ docFetcher.doc(iter.nextDoc());
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index 3512b0154ed..ff59b99f624 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -140,7 +140,7 @@ public class BlobHandler extends RequestHandlerBase
long version = 0;
if (docs.totalHits.value > 0) {
- Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+ Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc);
Number n = doc.getField("version").numericValue();
version = n.longValue();
}
@@ -214,7 +214,7 @@ public class BlobHandler extends RequestHandlerBase
@Override
public void write(OutputStream os) throws IOException {
- Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc);
+ Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc);
IndexableField sf = doc.getField("blob");
FieldType fieldType = req.getSchema().getField("blob").getType();
ByteBuffer buf = (ByteBuffer) fieldType.toObject(sf);
diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
index 2027012d23a..8b4a292b0a1 100644
--- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java
@@ -26,9 +26,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.mlt.MoreLikeThis;
import org.apache.lucene.search.BooleanClause;
@@ -282,10 +284,12 @@ public class MoreLikeThisHandler extends RequestHandlerBase {
}
/** Helper class for MoreLikeThis that can be called from other request handlers */
+ @NotThreadSafe
public static class MoreLikeThisHelper {
final SolrIndexSearcher searcher;
final MoreLikeThis mlt;
final IndexReader reader;
+ final StoredFields storedFields;
final SchemaField uniqueKeyField;
final boolean needDocSet;
Map<String, Float> boostFields;
@@ -293,6 +297,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase {
public MoreLikeThisHelper(SolrParams params, SolrIndexSearcher searcher) throws IOException {
this.searcher = searcher;
this.reader = searcher.getIndexReader();
+ this.storedFields = this.reader.storedFields();
this.uniqueKeyField = searcher.getSchema().getUniqueKeyField();
this.needDocSet = params.getBool(FacetParams.FACET, false);
@@ -394,7 +399,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase {
public DocListAndSet getMoreLikeThis(
int id, int start, int rows, List<Query> filters, int flags) throws IOException {
- Document doc = reader.document(id);
+ Document doc = this.storedFields.document(id);
final Query boostedQuery = getBoostedMLTQuery(id);
// exclude current document from results
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java
index b4f222a7a49..87d4764acba 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java
@@ -45,6 +45,8 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StandardDirectoryReader;
import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
@@ -384,11 +386,12 @@ public class IndexSizeEstimator {
for (LeafReaderContext leafReaderContext : reader.leaves()) {
LeafReader leafReader = leafReaderContext.reader();
Bits liveDocs = leafReader.getLiveDocs();
+ TermVectors leafTermVectors = leafReader.termVectors();
for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
if (liveDocs != null && !liveDocs.get(docId)) {
continue;
}
- Fields termVectors = leafReader.getTermVectors(docId);
+ Fields termVectors = leafTermVectors.get(docId);
if (termVectors == null) {
continue;
}
@@ -608,11 +611,12 @@ public class IndexSizeEstimator {
mergeInstance.close();
}
} else {
+ StoredFields storedFields = leafReader.storedFields();
for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
if (liveDocs != null && !liveDocs.get(docId)) {
continue;
}
- leafReader.document(docId, visitor);
+ storedFields.document(docId, visitor);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
index 318f6d0842b..c99c6f9afea 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
@@ -52,7 +52,9 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
@@ -161,7 +163,7 @@ public class LukeRequestHandler extends RequestHandlerBase {
}
Document doc = null;
try {
- doc = reader.document(docId);
+ doc = reader.storedFields().document(docId);
} catch (Exception ex) {
}
if (doc == null) {
@@ -316,6 +318,7 @@ public class LukeRequestHandler extends RequestHandlerBase {
Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException {
final CharsRefBuilder spare = new CharsRefBuilder();
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<>();
+ TermVectors termVectors = null;
for (Object o : doc.getFields()) {
Field field = (Field) o;
SimpleOrderedMap<Object> f = new SimpleOrderedMap<>();
@@ -353,7 +356,8 @@ public class LukeRequestHandler extends RequestHandlerBase {
// If we have a term vector, return that
if (field.fieldType().storeTermVectors()) {
try {
- Terms v = reader.getTermVector(docId, field.name());
+ if (termVectors == null) termVectors = reader.termVectors();
+ Terms v = termVectors.get(docId, field.name());
if (v != null) {
SimpleOrderedMap<Integer> tfv = new SimpleOrderedMap<>();
final TermsEnum termsEnum = v.iterator();
@@ -462,6 +466,7 @@ public class LukeRequestHandler extends RequestHandlerBase {
PostingsEnum postingsEnum = null;
TermsEnum termsEnum = terms.iterator();
BytesRef text;
+ StoredFields storedFields = reader.storedFields();
// Deal with the chance that the first bunch of terms are in deleted documents. Is there a
// better way?
for (int idx = 0; idx < 1000 && postingsEnum == null; ++idx) {
@@ -476,7 +481,7 @@ public class LukeRequestHandler extends RequestHandlerBase {
if (liveDocs != null && liveDocs.get(postingsEnum.docID())) {
continue;
}
- return reader.document(postingsEnum.docID());
+ return storedFields.document(postingsEnum.docID());
}
}
return null;
diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
index bc1267bfd54..2b41bdd1245 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
@@ -49,6 +49,7 @@ import org.apache.solr.search.DocList;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.QueryLimits;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.slf4j.Logger;
@@ -103,6 +104,7 @@ public class MoreLikeThisComponent extends SearchComponent {
new MoreLikeThisHandler.MoreLikeThisHelper(params, searcher);
NamedList<NamedList<?>> mltQueryByDocKey = new NamedList<>();
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
+ SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher();
for (DocIterator results = rb.getResults().docList.iterator(); results.hasNext(); ) {
int docId = results.nextDoc();
final List<MoreLikeThisHandler.InterestingTerm> interestingTerms =
@@ -114,7 +116,7 @@ public class MoreLikeThisComponent extends SearchComponent {
break;
}
final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName();
- final Document document = rb.req.getSearcher().doc(docId);
+ final Document document = docFetcher.doc(docId);
final String uniqueVal = rb.req.getSchema().printableUniqueKey(document);
final NamedList<String> mltQ =
mltViaQueryParams(rb.req.getSchema(), interestingTerms, uniqueKey, uniqueVal);
@@ -423,12 +425,13 @@ public class MoreLikeThisComponent extends SearchComponent {
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
while (iterator.hasNext()) {
int id = iterator.nextDoc();
int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5);
DocListAndSet similarDocuments = mltHelper.getMoreLikeThis(id, 0, rows, null, flags);
- String name = schema.printableUniqueKey(searcher.doc(id));
+ String name = schema.printableUniqueKey(docFetcher.doc(id));
mltResponse.add(name, similarDocuments.docList);
if (dbg != null) {
@@ -440,7 +443,7 @@ public class MoreLikeThisComponent extends SearchComponent {
DocIterator similarDocumentsIterator = similarDocuments.docList.iterator();
while (similarDocumentsIterator.hasNext()) {
int mltid = similarDocumentsIterator.nextDoc();
- String key = schema.printableUniqueKey(searcher.doc(mltid));
+ String key = schema.printableUniqueKey(docFetcher.doc(mltid));
explains.add(key, searcher.explain(mltHelper.getRealMLTQuery(), mltid));
}
docDbg.add("explain", explains);
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 33fa334758a..f1fc77d9ceb 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -347,10 +347,10 @@ public class RealTimeGetComponent extends SearchComponent {
if (docid < 0) continue;
+ SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
Document luceneDocument =
- searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames());
+ docFetcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames());
SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
- SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
if (reuseDvIters == null) {
reuseDvIters = new DocValuesIteratorCache(searcherInfo.getSearcher());
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java
index eba622615a3..6d8ee1957fb 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java
@@ -24,6 +24,7 @@ import org.apache.solr.response.ResultContext;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
/**
@@ -90,9 +91,10 @@ public class ResponseLogComponent extends SearchComponent {
StringBuilder sb = new StringBuilder();
Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
- sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(',');
+ sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))).append(',');
}
if (sb.length() > 0) {
rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
@@ -105,8 +107,9 @@ public class ResponseLogComponent extends SearchComponent {
StringBuilder sb = new StringBuilder();
Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
- sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields)))
+ sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields)))
.append(':')
.append(iter.score())
.append(',');
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
index 6373b0a6337..73eaa3b817b 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
@@ -34,6 +34,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
@@ -251,6 +252,7 @@ public class TermVectorComponent extends SearchComponent {
SolrIndexSearcher searcher = rb.req.getSearcher();
IndexReader reader = searcher.getIndexReader();
+ TermVectors termVectors = reader.termVectors();
// the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
// Only load the id field to get the uniqueKey of that
@@ -277,7 +279,7 @@ public class TermVectorComponent extends SearchComponent {
if (null != fields) {
for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
final String field = entry.getKey();
- final Terms vector = reader.getTermVector(docId, field);
+ final Terms vector = termVectors.get(docId, field);
if (vector != null) {
TermsEnum termsEnum = vector.iterator();
mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field);
@@ -285,7 +287,7 @@ public class TermVectorComponent extends SearchComponent {
}
} else {
// extract all fields
- final Fields vectors = reader.getTermVectors(docId);
+ final Fields vectors = termVectors.get(docId);
// There can be no documents with vectors
if (vectors != null) {
for (String field : vectors) {
diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
index 6fc2d475942..ba22b2e317c 100644
--- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
@@ -33,10 +33,9 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -78,6 +77,7 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.util.plugin.PluginInfoInitialized;
@@ -482,15 +482,16 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
// Lazy container for fvh and fieldQuery
FvhContainer fvhContainer = new FvhContainer(null, null);
- IndexReader reader =
- new TermVectorReusingLeafReader(req.getSearcher().getSlowAtomicReader()); // SOLR-5855
+ IndexReader reader = req.getSearcher().getSlowAtomicReader();
+ TermVectors termVectors = new ReusingTermVectors(reader); // SOLR-5855
// Highlight each document
NamedList<Object> fragments = new SimpleOrderedMap<>();
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++) {
int docId = iterator.nextDoc();
- SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, returnFields);
+ SolrDocument doc = docFetcher.solrDoc(docId, returnFields);
NamedList<Object> docHighlights = new SimpleOrderedMap<>();
// Highlight per-field
@@ -500,10 +501,11 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
Object fieldHighlights; // object type allows flexibility for subclassers
fieldHighlights =
doHighlightingOfField(
- doc, docId, schemaField, fvhContainer, query, reader, req, params);
+ doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params);
if (fieldHighlights == null) {
- fieldHighlights = alternateField(doc, docId, fieldName, fvhContainer, query, reader, req);
+ fieldHighlights =
+ alternateField(doc, docId, fieldName, fvhContainer, query, reader, termVectors, req);
}
if (fieldHighlights != null) {
@@ -522,6 +524,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
FvhContainer fvhContainer,
Query query,
IndexReader reader,
+ TermVectors termVectors,
SolrQueryRequest req,
SolrParams params)
throws IOException {
@@ -570,7 +573,8 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
fieldHighlights =
doHighlightingByFastVectorHighlighter(doc, docId, schemaField, fvhContainer, reader, req);
} else { // standard/default highlighter
- fieldHighlights = doHighlightingByHighlighter(doc, docId, schemaField, query, reader, req);
+ fieldHighlights =
+ doHighlightingByHighlighter(doc, docId, schemaField, query, termVectors, req);
}
return fieldHighlights;
}
@@ -656,7 +660,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
int docId,
SchemaField schemaField,
Query query,
- IndexReader reader,
+ TermVectors termVectors,
SolrQueryRequest req)
throws IOException {
final SolrParams params = req.getParams();
@@ -696,7 +700,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
// Try term vectors, which is faster
// note: offsets are minimally sufficient for this HL.
- final Fields tvFields = schemaField.storeTermOffsets() ? reader.getTermVectors(docId) : null;
+ final Fields tvFields = schemaField.storeTermOffsets() ? termVectors.get(docId) : null;
final TokenStream tvStream =
TokenSources.getTermVectorTokenStreamOrNull(fieldName, tvFields, maxCharsToAnalyze - 1);
// We need to wrap in OffsetWindowTokenFilter if multi-valued
@@ -858,6 +862,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
FvhContainer fvhContainer,
Query query,
IndexReader reader,
+ TermVectors termVectors,
SolrQueryRequest req)
throws IOException {
IndexSchema schema = req.getSearcher().getSchema();
@@ -887,7 +892,7 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
req.setParams(SolrParams.wrapDefaults(new MapSolrParams(invariants), origParams));
fieldHighlights =
doHighlightingOfField(
- doc, docId, schemaField, fvhContainer, query, reader, req, params);
+ doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params);
req.setParams(origParams);
if (fieldHighlights != null) {
return fieldHighlights;
@@ -1087,35 +1092,30 @@ public class DefaultSolrHighlighter extends SolrHighlighter implements PluginInf
}
/**
- * Wraps a DirectoryReader that caches the {@link LeafReader#getTermVectors(int)} so that if the
- * next call has the same ID, then it is reused.
+ * Wraps a TermVectors and caches the {@link TermVectors#get(int)} so that if the next call has
+ * the same ID, then it is reused.
*/
- static class TermVectorReusingLeafReader extends FilterLeafReader {
+ static class ReusingTermVectors extends TermVectors {
+ private final IndexReader reader;
+ private TermVectors in;
private int lastDocId = -1;
private Fields tvFields;
- public TermVectorReusingLeafReader(LeafReader in) {
- super(in);
+ public ReusingTermVectors(IndexReader reader) {
+ this.reader = reader;
}
@Override
- public Fields getTermVectors(int docID) throws IOException {
+ public Fields get(int docID) throws IOException {
if (docID != lastDocId) {
+ if (in == null) {
+ in = reader.termVectors();
+ }
lastDocId = docID;
- tvFields = in.getTermVectors(docID);
+ tvFields = in.get(docID);
}
return tvFields;
}
-
- @Override
- public CacheHelper getCoreCacheHelper() {
- return null;
- }
-
- @Override
- public CacheHelper getReaderCacheHelper() {
- return null;
- }
}
}
diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
index 4e7729fa49a..e106c471f63 100644
--- a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java
@@ -29,6 +29,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.util.SolrPluginUtils;
public abstract class SolrHighlighter {
@@ -73,7 +74,11 @@ public abstract class SolrHighlighter {
} else {
fields =
expandWildcardsInFields(
- () -> request.getSearcher().getDocFetcher().getStoredHighlightFieldNames(), fields);
+ () ->
+ request
+ .getSearcher()
+ .interrogateDocFetcher(SolrDocumentFetcher::getStoredHighlightFieldNames),
+ fields);
}
// Trim them now in case they haven't been yet. Not needed for all code-paths above but do it
diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
index f14c43d1fd3..4390893ec76 100644
--- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
@@ -49,6 +49,7 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.util.RTimerTree;
@@ -223,10 +224,11 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf
SchemaField keyField = schema.getUniqueKeyField();
if (keyField != null) {
SolrReturnFields returnFields = new SolrReturnFields(keyField.getName(), null);
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
String[] uniqueKeys = new String[docIDs.length];
for (int i = 0; i < docIDs.length; i++) {
int docid = docIDs[i];
- SolrDocument solrDoc = searcher.getDocFetcher().solrDoc(docid, returnFields);
+ SolrDocument solrDoc = docFetcher.solrDoc(docid, returnFields);
uniqueKeys[i] = schema.printableUniqueKey(solrDoc);
}
return uniqueKeys;
@@ -446,7 +448,8 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf
if (queryFieldPattern != null && queryFieldPattern.length != 0) {
Supplier<Collection<String>> indexedFieldsSupplier =
- () -> solrIndexSearcher.getDocFetcher().getIndexedFieldNames();
+ () ->
+ solrIndexSearcher.interrogateDocFetcher(SolrDocumentFetcher::getIndexedFieldNames);
Set<String> fields =
Set.of(expandWildcardsInFields(indexedFieldsSupplier, queryFieldPattern));
diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
index f8fffe9bce8..cdfbb181671 100644
--- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
+++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
@@ -313,6 +313,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
@Override
+ @Deprecated
public Fields getTermVectors(int docID) throws IOException {
return in.getTermVectors(docID);
}
@@ -342,6 +343,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
@Override
+ @Deprecated
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
ensureOpen();
in.document(docID, visitor);
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index 8fdcefa3d3f..485d9c9a1e8 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -73,7 +73,7 @@ public class DocsStreamer implements Iterator<SolrDocument> {
this.docs = rctx.getDocList();
transformer = rctx.getReturnFields().getTransformer();
docIterator = this.docs.iterator();
- docFetcher = rctx.getSearcher().getDocFetcher();
+ docFetcher = rctx.getDocFetcher();
solrReturnFields = (SolrReturnFields) rctx.getReturnFields();
if (transformer != null) transformer.setContext(rctx);
diff --git a/solr/core/src/java/org/apache/solr/response/ResultContext.java b/solr/core/src/java/org/apache/solr/response/ResultContext.java
index a74d0484d08..ea07d076da5 100644
--- a/solr/core/src/java/org/apache/solr/response/ResultContext.java
+++ b/solr/core/src/java/org/apache/solr/response/ResultContext.java
@@ -23,17 +23,28 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.DocList;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
/** A class to hold the QueryResult and the Query */
public abstract class ResultContext {
+ private SolrDocumentFetcher docFetcher;
+
public abstract DocList getDocList();
public abstract ReturnFields getReturnFields();
public abstract SolrIndexSearcher getSearcher();
+ public SolrDocumentFetcher getDocFetcher() {
+ if (docFetcher == null) {
+ return docFetcher = getSearcher().getDocFetcher();
+ } else {
+ return docFetcher;
+ }
+ }
+
public abstract Query getQuery();
// TODO: any reason to allow for retrieval of any filters as well?
diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
index 7eaac00e52c..d8a4f084226 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
@@ -45,6 +45,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.BitsFilteredPostingsEnum;
import org.apache.solr.search.DocSet;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.slf4j.Logger;
@@ -173,6 +174,7 @@ class ChildDocTransformer extends DocTransformer {
final Map<String, Map<String, List<SolrDocument>>> pendingParentPathsToChildren =
new HashMap<>();
+ SolrDocumentFetcher docFetcher = context.getDocFetcher();
final int firstChildId = segBaseId + segPrevRootId + 1;
int matches = 0;
// Loop each child ID up to the parent (exclusive).
@@ -207,7 +209,7 @@ class ChildDocTransformer extends DocTransformer {
++matches; // note: includes ancestors that are not necessarily in childDocSet
// load the doc
- SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, childReturnFields);
+ SolrDocument doc = docFetcher.solrDoc(docId, childReturnFields);
if (childReturnFields.getTransformer() != null) {
if (childReturnFields.getTransformer().context == null) {
childReturnFields.getTransformer().setContext(context);
diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
index e93b0fae772..44dcb12491e 100644
--- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java
+++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java
@@ -28,7 +28,7 @@ import org.apache.solr.response.transform.DocTransformer;
public abstract class ReturnFields {
/**
* Set of field names with their exact names from the lucene index. Class such as ResponseWriters
- * pass this to {@link SolrIndexSearcher#doc(int, Set)}.
+ * pass this to {@link SolrDocumentFetcher#doc(int, Set)}.
*
* <p>NOTE: In some situations, this method may return <code>null</code> even if {@link
* #wantsAllFields()} is <code>false</code>. For example: When glob expressions are used ({@link
diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
index e6ccb9edd18..ea7cf854aab 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
@@ -19,6 +19,7 @@ package org.apache.solr.search;
import java.io.IOException;
import java.io.Reader;
+import java.io.UncheckedIOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -42,7 +43,6 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
@@ -57,6 +57,7 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.misc.document.LazyDocument;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@@ -114,11 +115,38 @@ public class SolrDocumentFetcher {
private final Set<String> largeFields;
- private Collection<String> storedHighlightFieldNames; // lazy populated; use getter
+ private final Collection<String>[] storedHighlightFieldNames; // lazy populated; use getter
+
+ private final Collection<String>[] indexedFieldNames; // lazy populated; use getter
+
+ private final StoredFields storedFields;
+
+ private SolrDocumentFetcher(SolrDocumentFetcher template, StoredFields storedFields) {
+ this.searcher = template.searcher;
+ this.nLeaves = template.nLeaves;
+ this.enableLazyFieldLoading = template.enableLazyFieldLoading;
+ this.documentCache = template.documentCache;
+ this.nonStoredDVsUsedAsStored = template.nonStoredDVsUsedAsStored;
+ this.allNonStoredDVs = template.allNonStoredDVs;
+ this.nonStoredDVsWithoutCopyTargets = template.nonStoredDVsWithoutCopyTargets;
+ this.largeFields = template.largeFields;
+ this.dvsCanSubstituteStored = template.dvsCanSubstituteStored;
+ this.allStored = template.allStored;
+ this.storedHighlightFieldNames = template.indexedFieldNames;
+ this.indexedFieldNames = template.indexedFieldNames;
+ this.storedFields = storedFields;
+ }
- private Collection<String> indexedFieldNames; // lazy populated; use getter
+ @Override
+ protected SolrDocumentFetcher clone() {
+ try {
+ return new SolrDocumentFetcher(this, searcher.getIndexReader().storedFields());
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
- @SuppressWarnings({"unchecked"})
+ @SuppressWarnings({"unchecked", "rawtypes"})
SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) {
this.searcher = searcher;
this.nLeaves = searcher.getTopReaderContext().leaves().size();
@@ -172,6 +200,9 @@ public class SolrDocumentFetcher {
this.largeFields = Collections.unmodifiableSet(storedLargeFields);
this.dvsCanSubstituteStored = Collections.unmodifiableSet(dvsCanSubstituteStored);
this.allStored = Collections.unmodifiableSet(allStoreds);
+ this.storedFields = null; // template docFetcher should throw NPE if used directly
+ this.storedHighlightFieldNames = new Collection[1];
+ this.indexedFieldNames = new Collection[1];
}
// Does this field have both stored=true and docValues=true and is otherwise
@@ -203,9 +234,9 @@ public class SolrDocumentFetcher {
* reader knows about.
*/
public Collection<String> getStoredHighlightFieldNames() {
- synchronized (this) {
- if (storedHighlightFieldNames == null) {
- storedHighlightFieldNames = new ArrayList<>();
+ synchronized (storedHighlightFieldNames) {
+ if (storedHighlightFieldNames[0] == null) {
+ Collection<String> storedHighlightFieldNames = new ArrayList<>();
for (FieldInfo fieldInfo : searcher.getFieldInfos()) {
final String fieldName = fieldInfo.name;
try {
@@ -220,23 +251,25 @@ public class SolrDocumentFetcher {
log.warn("Field [{}] found in index, but not defined in schema.", fieldName);
}
}
+ this.storedHighlightFieldNames[0] = storedHighlightFieldNames;
}
- return storedHighlightFieldNames;
+ return storedHighlightFieldNames[0];
}
}
/** Returns a collection of the names of all indexed fields which the index reader knows about. */
public Collection<String> getIndexedFieldNames() {
- synchronized (this) {
- if (indexedFieldNames == null) {
- indexedFieldNames = new ArrayList<>();
+ synchronized (indexedFieldNames) {
+ if (indexedFieldNames[0] == null) {
+ Collection<String> indexedFieldNames = new ArrayList<>();
for (FieldInfo fieldInfo : searcher.getFieldInfos()) {
if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
indexedFieldNames.add(fieldInfo.name);
}
}
+ this.indexedFieldNames[0] = indexedFieldNames;
}
- return indexedFieldNames;
+ return indexedFieldNames[0];
}
}
@@ -272,10 +305,9 @@ public class SolrDocumentFetcher {
}
private Document docNC(int i, Set<String> fields) throws IOException {
- final DirectoryReader reader = searcher.getIndexReader();
final SolrDocumentStoredFieldVisitor visitor =
- new SolrDocumentStoredFieldVisitor(fields, reader, i);
- reader.document(i, visitor);
+ new SolrDocumentStoredFieldVisitor(fields, searcher.getIndexReader(), i);
+ storedFields.document(i, visitor);
return visitor.getDocument();
}
@@ -368,16 +400,14 @@ public class SolrDocumentFetcher {
}
}
- /**
- * @see SolrIndexSearcher#doc(int, StoredFieldVisitor)
- */
+ /** Visit a document's fields using a {@link StoredFieldVisitor}. */
public void doc(int docId, StoredFieldVisitor visitor) throws IOException {
if (documentCache != null) {
// get cached document or retrieve it including all fields (and cache it)
Document cached = doc(docId);
visitFromCached(cached, visitor);
} else {
- searcher.getIndexReader().document(docId, visitor);
+ storedFields.document(docId, visitor);
}
}
@@ -493,35 +523,33 @@ public class SolrDocumentFetcher {
return cachedBytes;
} else {
BytesRef bytesRef = new BytesRef();
- searcher
- .getIndexReader()
- .document(
- docId,
- new StoredFieldVisitor() {
- boolean done = false;
-
- @Override
- public Status needsField(FieldInfo fieldInfo) throws IOException {
- if (done) {
- return Status.STOP;
- }
- return fieldInfo.name.equals(name()) ? Status.YES : Status.NO;
- }
-
- @Override
- public void stringField(FieldInfo fieldInfo, String value) throws IOException {
- Objects.requireNonNull(value, "String value should not be null");
- bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8);
- bytesRef.length = bytesRef.bytes.length;
- done = true;
- }
-
- @Override
- public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
- throw new UnsupportedOperationException(
- "'large' binary fields are not (yet) supported");
- }
- });
+ storedFields.document(
+ docId,
+ new StoredFieldVisitor() {
+ boolean done = false;
+
+ @Override
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ if (done) {
+ return Status.STOP;
+ }
+ return fieldInfo.name.equals(name()) ? Status.YES : Status.NO;
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ Objects.requireNonNull(value, "String value should not be null");
+ bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8);
+ bytesRef.length = bytesRef.bytes.length;
+ done = true;
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+ throw new UnsupportedOperationException(
+ "'large' binary fields are not (yet) supported");
+ }
+ });
if (bytesRef.length < largeValueLengthCacheThreshold) {
return cachedBytes = bytesRef;
} else {
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index d4d1f64bcda..a8bc38b5699 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.LongAdder;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -427,7 +428,16 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
}
public SolrDocumentFetcher getDocFetcher() {
- return docFetcher;
+ return docFetcher.clone();
+ }
+
+ /**
+ * Allows interrogation of {@link #docFetcher} template (checking field names, etc.) without
+ * forcing it to be cloned (as it would be if an instance were retrieved via {@link
+ * #getDocFetcher()}).
+ */
+ public <T> T interrogateDocFetcher(Function<SolrDocumentFetcher, T> func) {
+ return func.apply(docFetcher);
}
public StatsCache getStatsCache() {
@@ -755,6 +765,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
* @see SolrDocumentFetcher
*/
@Override
+ @Deprecated
public Document doc(int docId) throws IOException {
return doc(docId, (Set<String>) null);
}
@@ -767,6 +778,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
* @see SolrDocumentFetcher
*/
@Override
+ @Deprecated
public final void doc(int docId, StoredFieldVisitor visitor) throws IOException {
getDocFetcher().doc(docId, visitor);
}
@@ -780,6 +792,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
* @see SolrDocumentFetcher
*/
@Override
+ @Deprecated
public final Document doc(int i, Set<String> fields) throws IOException {
return getDocFetcher().doc(i, fields);
}
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
index e385fcb941d..83255675fdd 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
@@ -43,6 +43,7 @@ import org.apache.solr.handler.component.ShardDoc;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.grouping.Command;
import org.apache.solr.search.grouping.distributed.command.QueryCommand;
import org.apache.solr.search.grouping.distributed.command.QueryCommandResult;
@@ -229,12 +230,13 @@ public class TopGroupsResultTransformer
groupResult.add("maxScore", searchGroup.maxScore);
}
+ SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher();
List<NamedList<Object>> documents = new ArrayList<>();
for (int i = 0; i < searchGroup.scoreDocs.length; i++) {
NamedList<Object> document = new NamedList<>();
documents.add(document);
- Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc);
+ Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc, docFetcher);
document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName())));
if (!Float.isNaN(searchGroup.scoreDocs[i].score)) {
document.add("score", searchGroup.scoreDocs[i].score);
@@ -290,13 +292,14 @@ public class TopGroupsResultTransformer
List<NamedList<?>> documents = new ArrayList<>();
queryResult.add("documents", documents);
+ SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher();
final IndexSchema schema = rb.req.getSearcher().getSchema();
SchemaField uniqueField = schema.getUniqueKeyField();
for (ScoreDoc scoreDoc : result.getTopDocs().scoreDocs) {
NamedList<Object> document = new NamedList<>();
documents.add(document);
- Document doc = retrieveDocument(uniqueField, scoreDoc.doc);
+ Document doc = retrieveDocument(uniqueField, scoreDoc.doc, docFetcher);
document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName())));
if (!Float.isNaN(scoreDoc.score)) {
document.add("score", scoreDoc.score);
@@ -322,7 +325,8 @@ public class TopGroupsResultTransformer
return queryResult;
}
- private Document retrieveDocument(final SchemaField uniqueField, int doc) throws IOException {
- return rb.req.getSearcher().doc(doc, Collections.singleton(uniqueField.getName()));
+ private Document retrieveDocument(
+ final SchemaField uniqueField, int doc, SolrDocumentFetcher docFetcher) throws IOException {
+ return docFetcher.doc(doc, Collections.singleton(uniqueField.getName()));
}
}
diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
index 3b564972eb7..a76ebe237a3 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
@@ -76,6 +76,7 @@ import org.apache.solr.search.FieldParams;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrQueryParser;
import org.apache.solr.search.SortSpecParsing;
@@ -249,9 +250,10 @@ public class SolrPluginUtils {
}
// get documents
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
DocIterator iter = docs.iterator();
for (int i = 0; i < docs.size(); i++) {
- searcher.doc(iter.nextDoc(), fieldFilter);
+ docFetcher.doc(iter.nextDoc(), fieldFilter);
}
}
}
@@ -404,12 +406,13 @@ public class SolrPluginUtils {
Query query, DocList docs, SolrIndexSearcher searcher, IndexSchema schema)
throws IOException {
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
NamedList<Explanation> explainList = new SimpleOrderedMap<>();
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++) {
int id = iterator.nextDoc();
- Document doc = searcher.doc(id);
+ Document doc = docFetcher.doc(id);
String strid = schema.printableUniqueKey(doc);
explainList.add(strid, searcher.explain(query, id));
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index ffb6f69ee29..f92fbe71e03 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -56,6 +56,7 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.PointRangeQuery;
@@ -5811,8 +5812,9 @@ public class TestPointFields extends SolrTestCaseJ4 {
}
for (LeafReaderContext leave : ir.leaves()) {
LeafReader reader = leave.reader();
+ StoredFields storedFields = reader.storedFields();
for (int i = 0; i < reader.numDocs(); i++) {
- Document doc = reader.document(i);
+ Document doc = storedFields.document(i);
if (sf.stored()) {
assertNotNull("Field " + field + " not found. Doc: " + doc, doc.get(field));
} else {
diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
index 901a188fedd..a554ce2f409 100644
--- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
@@ -303,6 +303,7 @@ public class TestDocSet extends SolrTestCase {
}
@Override
+ @Deprecated
public Fields getTermVectors(int docID) {
return null;
}
@@ -394,6 +395,7 @@ public class TestDocSet extends SolrTestCase {
protected void doClose() {}
@Override
+ @Deprecated
public void document(int doc, StoredFieldVisitor visitor) {}
@Override
diff --git a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java
index a02b55cb92f..2fc620f8468 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java
@@ -347,7 +347,7 @@ public class TestStressLucene extends TestRTGBase {
verbose("ERROR: Couldn't find a doc for id", id, "using reader", r);
}
assertTrue(docid >= 0); // we should have found the document, or its tombstone
- Document doc = r.document(docid);
+ Document doc = r.storedFields().document(docid);
long foundVal = Long.parseLong(doc.get(FIELD));
if (foundVal < Math.abs(val)) {
verbose(
diff --git a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java
index 5c37805b9e4..4081fc3b671 100644
--- a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java
+++ b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java
@@ -27,6 +27,7 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.IndexSearcher;
@@ -137,9 +138,10 @@ public class TestOrdValues extends SolrTestCase {
TopDocs td = s.search(q, 1000);
assertEquals("All docs should be matched!", N_DOCS, td.totalHits.value);
ScoreDoc sd[] = td.scoreDocs;
+ StoredFields storedFields = s.getIndexReader().storedFields();
for (int i = 0; i < sd.length; i++) {
float score = sd[i].score;
- String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
+ String id = storedFields.document(sd[i].doc).get(ID_FIELD);
log("-------- " + i + ". Explain doc " + id);
log(s.explain(q, sd[i].doc));
float expectedScore = N_DOCS - i - 1;
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
index 89ae4181f85..ec1525cef7d 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
@@ -38,6 +38,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
@@ -174,8 +175,9 @@ public class TestFieldCacheVsDocValues extends SolrTestCase {
TestUtil.checkReader(ar);
BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field");
+ StoredFields storedFields = ar.storedFields();
for (int docID = 0; docID < docBytes.size(); docID++) {
- Document doc = ar.document(docID);
+ Document doc = storedFields.document(docID);
assertEquals(docID, s.nextDoc());
BytesRef bytes = s.binaryValue();
byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
diff --git a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java
index d18e1b7c924..27646190a39 100644
--- a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java
+++ b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java
@@ -55,6 +55,7 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSlice;
import org.apache.solr.search.QueryLimits;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.carrot2.clustering.Cluster;
@@ -415,13 +416,14 @@ public class ClusteringComponent extends SearchComponent implements SolrCoreAwar
docLanguage = (doc) -> requestParameters.language();
}
+ SolrDocumentFetcher docFetcher = indexSearcher.getDocFetcher();
List<InputDocument> result = new ArrayList<>();
DocIterator it = responseBuilder.getResults().docList.iterator();
while (it.hasNext()) {
int docId = it.nextDoc();
Map<String, String> docFieldValues = new LinkedHashMap<>();
- for (IndexableField indexableField : indexSearcher.doc(docId, fieldsToLoad.keySet())) {
+ for (IndexableField indexableField : docFetcher.doc(docId, fieldsToLoad.keySet())) {
String fieldName = indexableField.name();
Function<IndexableField, String> toString = fieldsToLoad.get(fieldName);
if (toString != null) {
diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java
index 583e080954c..02e2fabb5bf 100644
--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java
+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java
@@ -29,6 +29,7 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -183,18 +184,21 @@ public class FieldValueFeature extends Feature {
public class FieldValueFeatureScorer extends FeatureScorer {
private final LeafReaderContext context;
+ private final StoredFields storedFields;
public FieldValueFeatureScorer(
- FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr) {
+ FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr)
+ throws IOException {
super(weight, itr);
this.context = context;
+ this.storedFields = (context == null ? null : context.reader().storedFields());
}
@Override
public float score() throws IOException {
try {
- final Document document = context.reader().document(itr.docID(), fieldAsSet);
+ final Document document = storedFields.document(itr.docID(), fieldAsSet);
final IndexableField indexableField = document.getField(field);
if (indexableField == null) {
return getDefaultValue();