Apache Lucene open-source search software
Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (#15929)
Zhou Huifa0e704d2a0f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 805fd682f5d..94b05c92f88 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -7,6 +7,7 @@ http://s.apache.org/luceneversionsAPI Changes---------------------+* GITHUB#15929: Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (Zhou Hui)* GITHUB#15763: Deprecate Operations.complement() method. This operation can be slow and is notrecommended for production use. It will be removed in Lucene 12. (Saurabh Singh)diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.mdindex 2a3eee9677d..f1189c6d8d0 100644--- a/lucene/MIGRATE.md+++ b/lucene/MIGRATE.md@@ -174,6 +174,10 @@ Automaton dfa = Operations.determinize(new RegExp(re).toAutomaton(), 10000);Query query = new AutomatonQuery(new Term("myfield", re), dfa);```+### CollectionStatistics and TermStatistics have been renamed to FieldStats and TermStats (GITHUB#15929)++Corresponding methods and parameters have been renamed accordingly.+## Migration from Lucene 10.4 to Lucene 10.5### `[Byte|Float]VectorSimilarityQuery` now performs adaptive HNSW graph traversaldiff --git a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.javaindex 1dcd0565b2f..5adb17d7e61 100644--- a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java@@ -52,8 +52,8 @@ import org.apache.lucene.util.SmallFloat;** <ol>* <li>Given a list of fields and weights, it pretends there is a synthetic combined field where- * all terms have been indexed. It computes new term and collection statistics for this- * combined field.+ * all terms have been indexed. It computes new term and field statistics for this combined+ * field.* <li>It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents.* </ol>*@@ -279,41 +279,37 @@ public final class CombinedFieldQuery extends Query implements Accountable {TermStates ts = TermStates.build(searcher, fieldTerms[i], true);termStates[i] = ts;if (ts.docFreq() > 0) {- TermStatistics termStats =- searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());+ TermStats termStats = searcher.termStats(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());docFreq = Math.max(termStats.docFreq(), docFreq);totalTermFreq += (double) field.weight * termStats.totalTermFreq();}}if (docFreq > 0) {- CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher);- TermStatistics pseudoTermStatistics =- new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));- this.simWeight =- searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics);+ FieldStats pseudoFieldStats = mergeFieldStats(searcher);+ TermStats pseudoTermStats =+ new TermStats(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));+ this.simWeight = searcher.getSimilarity().scorer(boost, pseudoFieldStats, pseudoTermStats);} else {this.simWeight = null;}}- private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher)- throws IOException {+ private FieldStats mergeFieldStats(IndexSearcher searcher) throws IOException {long maxDoc = 0;long docCount = 0;long sumTotalTermFreq = 0;long sumDocFreq = 0;for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {- CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field);- if (collectionStats != null) {- maxDoc = Math.max(collectionStats.maxDoc(), maxDoc);- docCount = Math.max(collectionStats.docCount(), docCount);- sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq);- sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq();+ FieldStats fieldStats = searcher.fieldStats(fieldWeight.field);+ if (fieldStats != null) {+ maxDoc = Math.max(fieldStats.maxDoc(), maxDoc);+ docCount = Math.max(fieldStats.docCount(), docCount);+ sumDocFreq = Math.max(fieldStats.sumDocFreq(), sumDocFreq);+ sumTotalTermFreq += (double) fieldWeight.weight * fieldStats.sumTotalTermFreq();}}- return new CollectionStatistics(- "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);+ return new FieldStats("pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java b/lucene/core/src/java/org/apache/lucene/search/FieldStats.javasimilarity index 90%rename from lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.javarename to lucene/core/src/java/org/apache/lucene/search/FieldStats.javaindex ba118fa5452..ca63bb75228 100644--- a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java+++ b/lucene/core/src/java/org/apache/lucene/search/FieldStats.java@@ -19,7 +19,7 @@ package org.apache.lucene.search;import java.util.Objects;/**- * Contains statistics for a collection (field).+ * Contains statistics for a field.** <p>This class holds statistics across all documents for scoring purposes:*@@ -54,22 +54,22 @@ import java.util.Objects;* Terms#getDocCount()* @param sumTotalTermFreq The total number of tokens for this field , in the range [{@link* #sumDocFreq()} .. {@link Long#MAX_VALUE}]. This is the "word count" for this field across all- * documents. It is the sum of {@link TermStatistics#totalTermFreq()} across all terms. It is- * also the sum of each document's field length across all documents.+ * documents. It is the sum of {@link TermStats#totalTermFreq()} across all terms. It is also+ * the sum of each document's field length across all documents.* <p>This value is always a positive number, and always at least {@link #sumDocFreq()}. @see* Terms#getSumTotalTermFreq()* @param sumDocFreq The total number of posting list entries for this field, in the range [{@link* #docCount()} .. {@link #sumTotalTermFreq()}]. This is the sum of term-document pairs: the sum- * of {@link TermStatistics#docFreq()} across all terms. It is also the sum of each document's- * unique term count for this field across all documents.+ * of {@link TermStats#docFreq()} across all terms. It is also the sum of each document's unique+ * term count for this field across all documents.* <p>This value is always a positive number, always at least {@link #docCount()}, and never* exceeds {@link #sumTotalTermFreq()}. @see Terms#getSumDocFreq()* @lucene.experimental*/-public record CollectionStatistics(+public record FieldStats(String field, long maxDoc, long docCount, long sumTotalTermFreq, long sumDocFreq) {/**- * Creates statistics instance for a collection (field).+ * Creates statistics instance for a field.** @throws IllegalArgumentException if {@code maxDoc} is negative or zero.* @throws IllegalArgumentException if {@code docCount} is negative or zero.@@ -77,7 +77,7 @@ public record CollectionStatistics(* @throws IllegalArgumentException if {@code sumDocFreq} is less than {@code docCount}.* @throws IllegalArgumentException if {@code sumTotalTermFreq} is less than {@code sumDocFreq}.*/- public CollectionStatistics {+ public FieldStats {Objects.requireNonNull(field);if (maxDoc <= 0) {throw new IllegalArgumentException("maxDoc must be positive, maxDoc: " + maxDoc);diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.javaindex b9d3f47cb09..845bae5238e 100644--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java@@ -1106,32 +1106,31 @@ public class IndexSearcher {}/**- * Returns {@link TermStatistics} for a term.+ * Returns {@link TermStats} for a term.** <p>This can be overridden for example, to return a term's statistics across a distributed* collection.** @param docFreq The document frequency of the term. It must be greater or equal to 1.* @param totalTermFreq The total term frequency.- * @return A {@link TermStatistics} (never null).+ * @return A {@link TermStats} (never null).* @lucene.experimental*/- public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)- throws IOException {+ public TermStats termStats(Term term, int docFreq, long totalTermFreq) throws IOException {// This constructor will throw an exception if docFreq <= 0.- return new TermStatistics(term.bytes(), docFreq, totalTermFreq);+ return new TermStats(term.bytes(), docFreq, totalTermFreq);}/**- * Returns {@link CollectionStatistics} for a field, or {@code null} if the field does not exist- * (has no indexed terms)+ * Returns {@link FieldStats} for a field, or {@code null} if the field does not exist (has no+ * indexed terms)** <p>This can be overridden for example, to return a field's statistics across a distributed* collection.** @lucene.experimental*/- public CollectionStatistics collectionStatistics(String field) throws IOException {+ public FieldStats fieldStats(String field) throws IOException {assert field != null;long docCount = 0;long sumTotalTermFreq = 0;@@ -1145,7 +1144,7 @@ public class IndexSearcher {if (docCount == 0) {return null;}- return new CollectionStatistics(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);+ return new FieldStats(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);}/**diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.javaindex 00b06901e5e..2812aad6440 100644--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java@@ -216,7 +216,7 @@ public class MultiPhraseQuery extends Query {protected Similarity.SimScorer getStats(IndexSearcher searcher) throws IOException {// compute idf- ArrayList<TermStatistics> allTermStats = new ArrayList<>();+ ArrayList<TermStats> allTermStats = new ArrayList<>();for (final Term[] terms : termArrays) {for (Term term : terms) {TermStates ts = termStates.get(term);@@ -225,7 +225,7 @@ public class MultiPhraseQuery extends Query {termStates.put(term, ts);}if (scoreMode.needsScores() && ts.docFreq() > 0) {- allTermStats.add(searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));+ allTermStats.add(searcher.termStats(term, ts.docFreq(), ts.totalTermFreq()));}}}@@ -233,9 +233,7 @@ public class MultiPhraseQuery extends Query {return null; // none of the terms were found, we won't use sim at all} else {return similarity.scorer(- boost,- searcher.collectionStatistics(field),- allTermStats.toArray(TermStatistics[]::new));+ boost, searcher.fieldStats(field), allTermStats.toArray(TermStats[]::new));}}diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.javaindex eea2a47a947..e8c9362629c 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java@@ -469,7 +469,7 @@ public class PhraseQuery extends Query {"PhraseWeight requires that the first position is 0, call rewrite first");}states = new TermStates[terms.length];- TermStatistics[] termStats = new TermStatistics[terms.length];+ TermStats[] termStats = new TermStats[terms.length];int termUpTo = 0;for (int i = 0; i < terms.length; i++) {final Term term = terms[i];@@ -477,16 +477,13 @@ public class PhraseQuery extends Query {if (scoreMode.needsScores()) {TermStates ts = states[i];if (ts.docFreq() > 0) {- termStats[termUpTo++] =- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq());+ termStats[termUpTo++] = searcher.termStats(term, ts.docFreq(), ts.totalTermFreq());}}}if (termUpTo > 0) {return similarity.scorer(- boost,- searcher.collectionStatistics(field),- ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));+ boost, searcher.fieldStats(field), ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));} else {return null; // no terms at all, we won't use similarity}diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.javaindex 8529f35f053..3ff634cd0e6 100644--- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java@@ -207,7 +207,7 @@ public final class SynonymQuery extends Query {super(query);assert scoreMode.needsScores();this.scoreMode = scoreMode;- CollectionStatistics collectionStats = searcher.collectionStatistics(field);+ FieldStats fieldStats = searcher.fieldStats(field);long docFreq = 0;long totalTermFreq = 0;termStates = new TermStates[terms.length];@@ -216,17 +216,16 @@ public final class SynonymQuery extends Query {TermStates ts = TermStates.build(searcher, term, true);termStates[i] = ts;if (ts.docFreq() > 0) {- TermStatistics termStats =- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq());+ TermStats termStats = searcher.termStats(term, ts.docFreq(), ts.totalTermFreq());docFreq = Math.max(termStats.docFreq(), docFreq);totalTermFreq += termStats.totalTermFreq();}}this.similarity = searcher.getSimilarity();if (docFreq > 0) {- TermStatistics pseudoStats =- new TermStatistics(new BytesRef("synonym pseudo-term"), docFreq, totalTermFreq);- this.simWeight = similarity.scorer(boost, collectionStats, pseudoStats);+ TermStats pseudoStats =+ new TermStats(new BytesRef("synonym pseudo-term"), docFreq, totalTermFreq);+ this.simWeight = similarity.scorer(boost, fieldStats, pseudoStats);} else {this.simWeight = null; // no terms exist at all, we won't use similarity}diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.javaindex 6d5cc2d9094..20b2d85ac16 100644--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java@@ -58,18 +58,18 @@ public class TermQuery extends Query {this.termStates = termStates;this.similarity = searcher.getSimilarity();- final CollectionStatistics collectionStats;- final TermStatistics termStats;+ final FieldStats fieldStats;+ final TermStats termStats;if (scoreMode.needsScores()) {- collectionStats = searcher.collectionStatistics(term.field());+ fieldStats = searcher.fieldStats(term.field());termStats =termStates.docFreq() > 0- ? searcher.termStatistics(term, termStates.docFreq(), termStates.totalTermFreq())+ ? searcher.termStats(term, termStates.docFreq(), termStates.totalTermFreq()): null;} else {// we do not need the actual stats, use fake stats with docFreq=maxDoc=ttf=1- collectionStats = new CollectionStatistics(term.field(), 1, 1, 1, 1);- termStats = new TermStatistics(term.bytes(), 1, 1);+ fieldStats = new FieldStats(term.field(), 1, 1, 1, 1);+ termStats = new TermStats(term.bytes(), 1, 1);}if (termStats == null) {@@ -79,7 +79,7 @@ public class TermQuery extends Query {// allocations in case default BM25Scorer is used.// See: https://github.com/apache/lucene/issues/12297if (scoreMode.needsScores()) {- this.simScorer = similarity.scorer(boost, collectionStats, termStats);+ this.simScorer = similarity.scorer(boost, fieldStats, termStats);} else {// Assigning a dummy scorer as this is not expected to be called since scores are not// needed.diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.javaindex 991b3a83171..4a7169a2e1f 100644--- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java@@ -83,7 +83,7 @@ public final class TermScorer extends Scorer {}/** Returns term frequency in the current document. */- public final int freq() throws IOException {+ public int freq() throws IOException {return postingsEnum.freq();}diff --git a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java b/lucene/core/src/java/org/apache/lucene/search/TermStats.javasimilarity index 83%rename from lucene/core/src/java/org/apache/lucene/search/TermStatistics.javarename to lucene/core/src/java/org/apache/lucene/search/TermStats.javaindex aeca533e9ba..639f66e0c31 100644--- a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java+++ b/lucene/core/src/java/org/apache/lucene/search/TermStats.java@@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef;* <ul>* <li>All statistics are positive integers: never zero or negative.* <li>{@code docFreq} <= {@code totalTermFreq}- * <li>{@code docFreq} <= {@code sumDocFreq} of the collection- * <li>{@code totalTermFreq} <= {@code sumTotalTermFreq} of the collection+ * <li>{@code docFreq} <= {@code sumDocFreq} of the field+ * <li>{@code totalTermFreq} <= {@code sumTotalTermFreq} of the field* </ul>** <p>Values may include statistics on deleted documents that have not yet been merged away.@@ -45,24 +45,23 @@ import org.apache.lucene.util.BytesRef;** @param term Term bytes.* <p>This value is never {@code null}.- * @param docFreq number of documents containing the term in the collection, in the range [1 ..- * {@link #totalTermFreq()}].+ * @param docFreq number of documents containing the term in the field, in the range [1 .. {@link+ * #totalTermFreq()}].* <p>This is the document-frequency for the term: the count of documents where the term appears* at least one time.* <p>This value is always a positive number, and never exceeds {@link #totalTermFreq}. It also- * cannot exceed {@link CollectionStatistics#sumDocFreq()}. @see TermsEnum#docFreq()- * @param totalTermFreq number of occurrences of the term in the collection, in the range [{@link- * #docFreq()} .. {@link CollectionStatistics#sumTotalTermFreq()}].+ * cannot exceed {@link FieldStats#sumDocFreq()}. @see TermsEnum#docFreq()+ * @param totalTermFreq number of occurrences of the term in the field, in the range [{@link+ * #docFreq()} .. {@link FieldStats#sumTotalTermFreq()}].* <p>This is the token count for the term: the number of times it appears in the field across* all documents.* <p>This value is always a positive number, always at least {@link #docFreq()}, and never- * exceeds {@link CollectionStatistics#sumTotalTermFreq()}. @see TermsEnum#totalTermFreq()+ * exceeds {@link FieldStats#sumTotalTermFreq()}. @see TermsEnum#totalTermFreq()* @lucene.experimental*/-// TODO: actually add missing cross-checks to guarantee TermStatistics is in bounds of-// CollectionStatistics,-// otherwise many similarity functions will implode.-public record TermStatistics(BytesRef term, long docFreq, long totalTermFreq) {+// TODO: actually add missing cross-checks to guarantee TermStats is in bounds of+// FieldStats, otherwise many similarity functions will implode.+public record TermStats(BytesRef term, long docFreq, long totalTermFreq) {/*** Creates statistics instance for a term.*@@ -70,7 +69,7 @@ public record TermStatistics(BytesRef term, long docFreq, long totalTermFreq) {* @throws IllegalArgumentException if {@code docFreq} is negative or zero.* @throws IllegalArgumentException if {@code totalTermFreq} is less than {@code docFreq}.*/- public TermStatistics {+ public TermStats {Objects.requireNonNull(term);if (docFreq <= 0) {throw new IllegalArgumentException("docFreq must be positive, docFreq: " + docFreq);diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.javaindex e2a86481aeb..3264b265bdd 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;import java.util.ArrayList;import java.util.List;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.SmallFloat;@@ -141,8 +141,8 @@ public class BM25Similarity extends Similarity {}/** The default implementation computes the average as <code>sumTotalTermFreq / docCount</code> */- protected float avgFieldLength(CollectionStatistics collectionStats) {- return (float) (collectionStats.sumTotalTermFreq() / (double) collectionStats.docCount());+ protected float avgFieldLength(FieldStats fieldStats) {+ return (float) (fieldStats.sumTotalTermFreq() / (double) fieldStats.docCount());}/** Cache of decoded bytes. */@@ -163,20 +163,20 @@ public class BM25Similarity extends Similarity {* idf(docFreq, docCount);* </code></pre>*- * Note that {@link CollectionStatistics#docCount()} is used instead of {@link+ * Note that {@link FieldStats#docCount()} is used instead of {@link* org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also {@link- * TermStatistics#docFreq()} is used, and when the latter is inaccurate, so is {@link- * CollectionStatistics#docCount()}, and in the same direction. In addition, {@link- * CollectionStatistics#docCount()} does not skew when fields are sparse.+ * TermStats#docFreq()} is used, and when the latter is inaccurate, so is {@link+ * FieldStats#docCount()}, and in the same direction. In addition, {@link FieldStats#docCount()}+ * does not skew when fields are sparse.*- * @param collectionStats collection-level statistics+ * @param fieldStats field-level statistics* @param termStats term-level statistics for the term* @return an Explain object that includes both an idf score factor and an explanation for the* term.*/- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {final long df = termStats.docFreq();- final long docCount = collectionStats.docCount();+ final long docCount = fieldStats.docCount();final float idf = idf(df, docCount);return Explanation.match(idf,@@ -190,16 +190,16 @@ public class BM25Similarity extends Similarity {** <p>The default implementation sums the idf factor for each term in the phrase.*- * @param collectionStats collection-level statistics+ * @param fieldStats field-level statistics* @param termStats term-level statistics for the terms in the phrase* @return an Explain object that includes both an idf score factor for the phrase and an* explanation for each term.*/- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {double idf = 0d; // sum into a double before casting into a floatList<Explanation> details = new ArrayList<>();- for (final TermStatistics stat : termStats) {- Explanation idfExplain = idfExplain(collectionStats, stat);+ for (final TermStats stat : termStats) {+ Explanation idfExplain = idfExplain(fieldStats, stat);details.add(idfExplain);idf += idfExplain.getValue().floatValue();}@@ -207,13 +207,12 @@ public class BM25Similarity extends Similarity {}@Override- public final SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {Explanation idf =termStats.length == 1- ? idfExplain(collectionStats, termStats[0])- : idfExplain(collectionStats, termStats);- float avgdl = avgFieldLength(collectionStats);+ ? idfExplain(fieldStats, termStats[0])+ : idfExplain(fieldStats, termStats);+ float avgdl = avgFieldLength(fieldStats);float[] cache = new float[256];for (int i = 0; i < cache.length; i++) {@@ -222,7 +221,7 @@ public class BM25Similarity extends Similarity {return new BM25Scorer(boost, k1, b, idf, avgdl, cache);}- /** Collection statistics for the BM25 model. */+ /** Field statistics for the BM25 model. */private static class BM25Scorer extends SimScorer {/** query boost */private final float boost;diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.javaindex db28e845448..3f3bf798b11 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java@@ -16,9 +16,9 @@*/package org.apache.lucene.search.similarities;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/*** Simple similarity that gives terms a score that is equal to their query boost. This similarity is@@ -34,8 +34,7 @@ public class BooleanSimilarity extends Similarity {public BooleanSimilarity() {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {return new BooleanWeight(boost);}diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.javaindex 5fad406c585..bcb93d7f07f 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java@@ -16,9 +16,9 @@*/package org.apache.lucene.search.similarities;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/*** Expert: Historical scoring implementation. You might want to consider using {@link@@ -53,9 +53,9 @@ public class ClassicSimilarity extends TFIDFSimilarity {}@Override- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {final long df = termStats.docFreq();- final long docCount = collectionStats.docCount();+ final long docCount = fieldStats.docCount();final float idf = idf(df, docCount);return Explanation.match(idf,diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.javaindex 5bd48f37a34..b7a3f395dc7 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;import java.util.List;import java.util.Locale;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/*** Abstract superclass for language modeling Similarities. The following inner types are introduced:@@ -66,9 +66,8 @@ public abstract class LMSimilarity extends SimilarityBase {* Computes the collection probability of the current term in addition to the usual statistics.*/@Override- protected void fillBasicStats(- BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {- super.fillBasicStats(stats, collectionStats, termStats);+ protected void fillBasicStats(BasicStats stats, FieldStats fieldStats, TermStats termStats) {+ super.fillBasicStats(stats, fieldStats, termStats);LMStats lmStats = (LMStats) stats;lmStats.setCollectionProbability(collectionModel.computeProbability(stats));}diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.javaindex 22dea01d6f6..7de0e018028 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java@@ -19,9 +19,9 @@ package org.apache.lucene.search.similarities;import java.util.ArrayList;import java.util.List;import org.apache.lucene.index.FieldInvertState;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/*** Implements the CombSUM method for combining evidence from multiple similarity values described@@ -44,11 +44,10 @@ public class MultiSimilarity extends Similarity {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {SimScorer[] subScorers = new SimScorer[sims.length];for (int i = 0; i < subScorers.length; i++) {- subScorers[i] = sims[i].scorer(boost, collectionStats, termStats);+ subScorers[i] = sims[i].scorer(boost, fieldStats, termStats);}return new MultiSimScorer(subScorers);}diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.javaindex 2107b151031..3689787e114 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java@@ -17,8 +17,8 @@package org.apache.lucene.search.similarities;import org.apache.lucene.index.FieldInvertState;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/*** Provides the ability to use a different {@link Similarity} for different fields.@@ -39,9 +39,8 @@ public abstract class PerFieldSimilarityWrapper extends Similarity {}@Override- public final SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {- return get(collectionStats.field()).scorer(boost, collectionStats, termStats);+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {+ return get(fieldStats.field()).scorer(boost, fieldStats, termStats);}/** Returns a {@link Similarity} for scoring a field. */diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.javaindex 043d831dbe1..508bd54f621 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java@@ -16,8 +16,8 @@*/package org.apache.lucene.search.similarities;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;/** Similarity that returns the raw TF as score. */public class RawTFSimilarity extends Similarity {@@ -33,8 +33,7 @@ public class RawTFSimilarity extends Similarity {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {return new SimScorer() {@Overridepublic float score(float freq, long norm) {diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.javaindex dc39ac3f321..dad5da0ce7d 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java@@ -22,10 +22,10 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.index.FieldInvertState;import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.util.SmallFloat;/**@@ -52,8 +52,7 @@ import org.apache.lucene.util.SmallFloat;* single byte, this might not be suitable for all purposes.** <p>Many formulas require the use of average document length, which can be computed via a- * combination of {@link CollectionStatistics#sumTotalTermFreq()} and {@link- * CollectionStatistics#docCount()}.+ * combination of {@link FieldStats#sumTotalTermFreq()} and {@link FieldStats#docCount()}.** <p>Additional scoring factors can be stored in named {@link NumericDocValuesField}s and accessed* at query-time with {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}.@@ -70,13 +69,12 @@ import org.apache.lucene.util.SmallFloat;* steps:** <ol>- * <li>The {@link #scorer(float, CollectionStatistics, TermStatistics...)} method is called a- * single time, allowing the implementation to compute any statistics (such as IDF, average- * document length, etc) across <i>the entire collection</i>. The {@link TermStatistics} and- * {@link CollectionStatistics} passed in already contain all of the raw statistics involved,- * so a Similarity can freely use any combination of statistics without causing any additional- * I/O. Lucene makes no assumption about what is stored in the returned {@link- * Similarity.SimScorer} object.+ * <li>The {@link #scorer(float, FieldStats, TermStats...)} method is called a single time,+ * allowing the implementation to compute any statistics (such as IDF, average document+ * length, etc) across <i>the entire field</i>. The {@link TermStats} and {@link FieldStats}+ * passed in already contain all of the raw statistics involved, so a Similarity can freely+ * use any combination of statistics without causing any additional I/O. Lucene makes no+ * assumption about what is stored in the returned {@link Similarity.SimScorer} object.* <li>Then {@link SimScorer#score(float, long)} is called for every matching document to compute* its score.* </ol>@@ -179,23 +177,21 @@ public abstract class Similarity {}/**- * Compute any collection-level weight (e.g. IDF, average document length, etc) needed for scoring- * a query.+ * Compute any field-level weight (e.g. IDF, average document length, etc) needed for scoring a+ * query.** @param boost a multiplicative factor to apply to the produces scores- * @param collectionStats collection-level statistics, such as the number of tokens in the- * collection.+ * @param fieldStats field-level statistics, such as the number of tokens in the field.* @param termStats term-level statistics, such as the document frequency of a term across the- * collection.+ * field.* @return SimWeight object with the information this Similarity needs to score a query.*/- public abstract SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats);+ public abstract SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats);/**- * Stores the weight for a query across the indexed collection. This abstract implementation is- * empty; descendants of {@code Similarity} should subclass {@code SimWeight} and define the- * statistics they require in the subclass. Examples include idf, average field length, etc.+ * Stores the weight for a query across the indexed field. This abstract implementation is empty;+ * descendants of {@code Similarity} should subclass {@code SimWeight} and define the statistics+ * they require in the subclass. Examples include idf, average field length, etc.*/public abstract static class SimScorer {diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.javaindex af63b7bebeb..7ec2f50ff95 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;import java.util.ArrayList;import java.util.List;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.util.SmallFloat;/**@@ -52,12 +52,11 @@ public abstract class SimilarityBase extends Similarity {}@Override- public final SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {SimScorer[] weights = new SimScorer[termStats.length];for (int i = 0; i < termStats.length; i++) {- BasicStats stats = newStats(collectionStats.field(), boost);- fillBasicStats(stats, collectionStats, termStats[i]);+ BasicStats stats = newStats(fieldStats.field(), boost);+ fillBasicStats(stats, fieldStats, termStats[i]);weights[i] = new BasicSimScorer(stats);}if (weights.length == 1) {@@ -76,17 +75,15 @@ public abstract class SimilarityBase extends Similarity {* Fills all member fields defined in {@code BasicStats} in {@code stats}. Subclasses can override* this method to fill additional stats.*/- protected void fillBasicStats(- BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {+ protected void fillBasicStats(BasicStats stats, FieldStats fieldStats, TermStats termStats) {// TODO: validate this for real, somewhere else- assert termStats.totalTermFreq() <= collectionStats.sumTotalTermFreq();- assert termStats.docFreq() <= collectionStats.sumDocFreq();+ assert termStats.totalTermFreq() <= fieldStats.sumTotalTermFreq();+ assert termStats.docFreq() <= fieldStats.sumDocFreq();// TODO: add sumDocFreq for field (numberOfFieldPostings)- stats.setNumberOfDocuments(collectionStats.docCount());- stats.setNumberOfFieldTokens(collectionStats.sumTotalTermFreq());- stats.setAvgFieldLength(- collectionStats.sumTotalTermFreq() / (double) collectionStats.docCount());+ stats.setNumberOfDocuments(fieldStats.docCount());+ stats.setNumberOfFieldTokens(fieldStats.sumTotalTermFreq());+ stats.setAvgFieldLength(fieldStats.sumTotalTermFreq() / (double) fieldStats.docCount());stats.setDocFreq(termStats.docFreq());stats.setTotalTermFreq(termStats.totalTermFreq());}diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.javaindex 47385de6583..04b18870e1f 100644--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java@@ -18,10 +18,10 @@ package org.apache.lucene.search.similarities;import java.util.ArrayList;import java.util.List;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.util.SmallFloat;/**@@ -357,20 +357,20 @@ public abstract class TFIDFSimilarity extends Similarity {* idf(docFreq, docCount);* </code></pre>*- * Note that {@link CollectionStatistics#docCount()} is used instead of {@link+ * Note that {@link FieldStats#docCount()} is used instead of {@link* org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also {@link- * TermStatistics#docFreq()} is used, and when the latter is inaccurate, so is {@link- * CollectionStatistics#docCount()}, and in the same direction. In addition, {@link- * CollectionStatistics#docCount()} does not skew when fields are sparse.+ * TermStats#docFreq()} is used, and when the latter is inaccurate, so is {@link+ * FieldStats#docCount()}, and in the same direction. In addition, {@link FieldStats#docCount()}+ * does not skew when fields are sparse.*- * @param collectionStats collection-level statistics+ * @param fieldStats collection-level statistics* @param termStats term-level statistics for the term* @return an Explain object that includes both an idf score factor and an explanation for the* term.*/- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {final long df = termStats.docFreq();- final long docCount = collectionStats.docCount();+ final long docCount = fieldStats.docCount();final float idf = idf(df, docCount);return Explanation.match(idf,@@ -384,16 +384,16 @@ public abstract class TFIDFSimilarity extends Similarity {** <p>The default implementation sums the idf factor for each term in the phrase.*- * @param collectionStats collection-level statistics+ * @param fieldStats collection-level statistics* @param termStats term-level statistics for the terms in the phrase* @return an Explain object that includes both an idf score factor for the phrase and an* explanation for each term.*/- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {double idf = 0d; // sum into a double before casting into a floatList<Explanation> subs = new ArrayList<>();- for (final TermStatistics stat : termStats) {- Explanation idfExplain = idfExplain(collectionStats, stat);+ for (final TermStats stat : termStats) {+ Explanation idfExplain = idfExplain(fieldStats, stat);subs.add(idfExplain);idf += idfExplain.getValue().floatValue();}@@ -433,12 +433,11 @@ public abstract class TFIDFSimilarity extends Similarity {}@Override- public final SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {final Explanation idf =termStats.length == 1- ? idfExplain(collectionStats, termStats[0])- : idfExplain(collectionStats, termStats);+ ? idfExplain(fieldStats, termStats[0])+ : idfExplain(fieldStats, termStats);float[] normTable = new float[256];for (int i = 1; i < 256; ++i) {float norm = lengthNorm(LENGTH_TABLE[i]);@@ -448,10 +447,7 @@ public abstract class TFIDFSimilarity extends Similarity {return new TFIDFScorer(boost, idf, normTable);}- /**- * Collection statistics for the TF-IDF model. The only statistic of interest to this model is- * idf.- */+ /** Field statistics for the TF-IDF model. The only statistic of interest to this model is idf. */class TFIDFScorer extends SimScorer {/** The idf and its explanation */private final Explanation idf;diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.javaindex 705c905012b..9a40aab030b 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java@@ -22,8 +22,8 @@ import java.util.stream.IntStream;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.TextField;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.ClassicSimilarity;import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;import org.apache.lucene.search.similarities.Similarity;@@ -105,8 +105,7 @@ public class TestCustomNorms extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.javaindex e42f3387630..fe76b2b3af0 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java@@ -26,8 +26,8 @@ import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.TextField;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -493,8 +493,7 @@ public class TestCustomTermFreq extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.javaindex 378388d86cb..eda73d9ae82 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java@@ -22,8 +22,8 @@ import java.util.Map;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.TextField;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.CannedTokenStream;@@ -50,8 +50,7 @@ public class TestFieldInvertState extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.javaindex 09d097a56c2..123d211f6db 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java@@ -63,9 +63,9 @@ import org.apache.lucene.document.SortedSetDocValuesField;import org.apache.lucene.document.StoredField;import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.FieldDoc;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.search.Query;@@ -77,7 +77,7 @@ import org.apache.lucene.search.SortedNumericSortField;import org.apache.lucene.search.SortedSetSelector;import org.apache.lucene.search.SortedSetSortField;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.TopFieldCollectorManager;import org.apache.lucene.search.similarities.Similarity;@@ -2295,9 +2295,8 @@ public class TestIndexSorting extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {- return in.scorer(boost, collectionStats, termStats);+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {+ return in.scorer(boost, fieldStats, termStats);}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.javaindex 7715045549c..55d92cf0963 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java@@ -22,8 +22,8 @@ import java.util.Collections;import java.util.List;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -102,8 +102,7 @@ public class TestMaxTermFrequency extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {return new SimScorer() {@Overridediff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.javaindex e5a887a8821..5a25b18f592 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java@@ -24,8 +24,8 @@ import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.Field.Store;import org.apache.lucene.document.TextField;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.ClassicSimilarity;import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;import org.apache.lucene.search.similarities.Similarity;@@ -108,8 +108,7 @@ public class TestNorms extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.javaindex 2b59333c5ee..d236f091b9f 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java@@ -25,16 +25,16 @@ import org.apache.lucene.document.FieldType;import org.apache.lucene.document.TextField;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.CollectorManager;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.PhraseQuery;import org.apache.lucene.search.Scorable;import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.SimpleCollector;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.TFIDFSimilarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -61,8 +61,7 @@ public class TestOmitTf extends LuceneTestCase {}@Override- public Explanation idfExplain(- CollectionStatistics collectionStats, TermStatistics[] termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {return Explanation.match(1.0f, "Inexplicable");}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.javaindex 234ea194779..2d3b619a4a5 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java@@ -20,8 +20,8 @@ import java.util.ArrayList;import java.util.HashSet;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;-import org.apache.lucene.search.CollectionStatistics;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -100,8 +100,7 @@ public class TestUniqueTermCount extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.javaindex df30c5ee005..3b68c094a99 100644--- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java+++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java@@ -188,8 +188,7 @@ final class JustCompileSearch {static final class JustCompileSimilarity extends Similarity {@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException(UNSUPPORTED_MSG);}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.javaindex 325d1344d62..f32627af784 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java@@ -480,13 +480,12 @@ public class TestCombinedFieldQuery extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {- return new BM25Similarity().scorer(boost, collectionStats, termStats);+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {+ return new BM25Similarity().scorer(boost, fieldStats, termStats);}}- public void testOverrideCollectionStatistics() throws IOException {+ public void testOverrideFieldStats() throws IOException {Directory dir = newDirectory();IndexWriterConfig iwc = new IndexWriterConfig();Similarity similarity = randomCompatibleSimilarity();@@ -533,8 +532,8 @@ public class TestCombinedFieldQuery extends LuceneTestCase {IndexSearcher searcher =new IndexSearcher(reader) {@Override- public CollectionStatistics collectionStatistics(String field) throws IOException {- CollectionStatistics shardStatistics = super.collectionStatistics(field);+ public FieldStats fieldStats(String field) throws IOException {+ FieldStats shardStatistics = super.fieldStats(field);int extraSumTotalTermFreq;if (field.equals("a")) {extraSumTotalTermFreq = extraSumTotalTermFreqA;@@ -545,7 +544,7 @@ public class TestCombinedFieldQuery extends LuceneTestCase {} else {throw new AssertionError("should never be called");}- return new CollectionStatistics(+ return new FieldStats(field,shardStatistics.maxDoc() + extraMaxDoc,shardStatistics.docCount() + extraDocCount,diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.javaindex e06951df164..0a1269dd8c7 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java@@ -371,8 +371,8 @@ public class TestMinShouldMatch2 extends LuceneTestCase {sims[(int) ord] =weight.similarity.scorer(1f,- searcher.collectionStatistics("field"),- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));+ searcher.fieldStats("field"),+ searcher.termStats(term, ts.docFreq(), ts.totalTermFreq()));}}norms = reader.getNormValues("field");diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.javaindex 1916efeee0f..c178e65c78d 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java@@ -49,7 +49,7 @@ public class TestSimilarity extends LuceneTestCase {}@Override- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] stats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] stats) {return Explanation.match(1.0f, "Inexplicable");}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.javaindex afdbef97b49..30580f18c78 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java@@ -109,8 +109,7 @@ public class TestSimilarityProvider extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {return new SimScorer() {@Override@@ -129,8 +128,7 @@ public class TestSimilarityProvider extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {return new SimScorer() {@Overridepublic float score(float freq, long norm) {diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.javaindex f87acb83bdf..33888b3a902 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java@@ -194,10 +194,9 @@ public class TestTermQuery extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {scorerCalled.set(true);- return existingSimilarity.scorer(boost, collectionStats, termStats);+ return existingSimilarity.scorer(boost, fieldStats, termStats);}});TermQuery termQuery =diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.javaindex 3ec08d75f85..be7c18e3919 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java@@ -157,7 +157,7 @@ public class TestClassicSimilarity extends BaseSimilarityTestCase {public void testSaneNormValues() throws IOException {ClassicSimilarity sim = new ClassicSimilarity();TFIDFSimilarity.TFIDFScorer stats =- (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.collectionStatistics("test"));+ (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.fieldStats("test"));for (int i = 0; i < 256; i++) {float boost = stats.normTable[i];assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.javaindex af2cf4c8268..0083069001f 100644--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java@@ -27,12 +27,12 @@ import org.apache.lucene.index.FieldInvertState;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Term;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.similarities.SimilarityBase.BasicSimScorer;import org.apache.lucene.store.Directory;@@ -187,7 +187,7 @@ public class TestSimilarityBase extends LuceneTestCase {return stats;}- private CollectionStatistics toCollectionStats(BasicStats stats) {+ private FieldStats toFieldStats(BasicStats stats) {long sumTtf = stats.getNumberOfFieldTokens();long sumDf;if (sumTtf == -1) {@@ -200,12 +200,11 @@ public class TestSimilarityBase extends LuceneTestCase {int docCount = Math.toIntExact(Math.min(sumDf, stats.getNumberOfDocuments()));int maxDoc = TestUtil.nextInt(random(), docCount, docCount + 10);- return new CollectionStatistics(stats.field, maxDoc, docCount, sumTtf, sumDf);+ return new FieldStats(stats.field, maxDoc, docCount, sumTtf, sumDf);}- private TermStatistics toTermStats(BasicStats stats) {- return new TermStatistics(- new BytesRef("spoofyText"), stats.getDocFreq(), stats.getTotalTermFreq());+ private TermStats toTermStats(BasicStats stats) {+ return new TermStats(new BytesRef("spoofyText"), stats.getDocFreq(), stats.getTotalTermFreq());}/**@@ -217,8 +216,7 @@ public class TestSimilarityBase extends LuceneTestCase {for (SimilarityBase sim : sims) {BasicStats realStats =((BasicSimScorer)- sim.scorer(- (float) stats.getBoost(), toCollectionStats(stats), toTermStats(stats)))+ sim.scorer((float) stats.getBoost(), toFieldStats(stats), toTermStats(stats))).stats;float score = (float) sim.score(realStats, freq, docLen);float explScore =@@ -471,7 +469,7 @@ public class TestSimilarityBase extends LuceneTestCase {BasicStats stats = createStats();BasicStats realStats =((BasicSimScorer)- sim.scorer((float) stats.getBoost(), toCollectionStats(stats), toTermStats(stats)))+ sim.scorer((float) stats.getBoost(), toFieldStats(stats), toTermStats(stats))).stats;float score = (float) sim.score(realStats, FREQ, DOC_LEN);assertEquals(sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON);diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.javaindex 1a98699bb7b..0c6782d671f 100644--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java@@ -74,8 +74,8 @@ import org.apache.lucene.index.Term;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.index.VectorSimilarityFunction;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.KnnByteVectorQuery;import org.apache.lucene.search.KnnFloatVectorQuery;@@ -83,7 +83,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.search.PhraseQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.search.similarities.BM25Similarity;@@ -217,8 +217,7 @@ public class TestMemoryIndex extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}});diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.javaindex 4373d77d381..cf180d19c4a 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java@@ -23,9 +23,9 @@ import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.queries.function.FunctionValues;import org.apache.lucene.queries.function.ValueSource;import org.apache.lucene.queries.function.docvalues.FloatDocValues;-import org.apache.lucene.search.CollectionStatistics;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity.SimScorer;import org.apache.lucene.search.similarities.TFIDFSimilarity;import org.apache.lucene.util.BytesRef;@@ -73,9 +73,7 @@ public class NormValueSource extends ValueSource {// is 1 when docCount == docFreq == 1final SimScorer simScorer =similarity.scorer(- 1f,- new CollectionStatistics(field, 1, 1, 1, 1),- new TermStatistics(new BytesRef("bogus"), 1, 1));+ 1f, new FieldStats(field, 1, 1, 1, 1), new TermStats(new BytesRef("bogus"), 1, 1));final NumericDocValues norms = readerContext.reader().getNormValues(field);return new FloatDocValues(this) {diff --git a/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.javaindex 554a3669fc1..9916c4e80ef 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java@@ -61,8 +61,8 @@ import org.apache.lucene.search.ScoreMode;* teacherid 2 (which has a 'james' in position 0 and 'jones' in position 1).** <p>Note: as {@link #getField()} returns the masked field, scoring will be done using the- * Similarity and collection statistics of the field name supplied, but with the term statistics of- * the real field. This may lead to exceptions, poor performance, and unexpected scoring behaviour.+ * Similarity and field statistics of the field name supplied, but with the term statistics of the+ * real field. This may lead to exceptions, poor performance, and unexpected scoring behavior.*/public final class FieldMaskingSpanQuery extends SpanQuery {private final SpanQuery maskedQuery;diff --git a/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java b/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.javaindex 8c7d8dbdb3a..4d130dca258 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java@@ -26,8 +26,8 @@ import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.Term;import org.apache.lucene.index.TermStates;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Matches;import org.apache.lucene.search.MatchesIterator;@@ -35,7 +35,7 @@ import org.apache.lucene.search.MatchesUtils;import org.apache.lucene.search.Query;import org.apache.lucene.search.ScorerSupplier;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.Weight;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.search.similarities.Similarity.SimScorer;@@ -102,19 +102,18 @@ public abstract class SpanWeight extends Weight {SpanQuery query, IndexSearcher searcher, Map<Term, TermStates> termStates, float boost)throws IOException {if (termStates == null || termStates.size() == 0 || query.getField() == null) return null;- TermStatistics[] termStats = new TermStatistics[termStates.size()];+ TermStats[] termStats = new TermStats[termStates.size()];int termUpTo = 0;for (Map.Entry<Term, TermStates> entry : termStates.entrySet()) {TermStates ts = entry.getValue();if (ts.docFreq() > 0) {termStats[termUpTo++] =- searcher.termStatistics(entry.getKey(), ts.docFreq(), ts.totalTermFreq());+ searcher.termStats(entry.getKey(), ts.docFreq(), ts.totalTermFreq());}}- CollectionStatistics collectionStats = searcher.collectionStatistics(query.getField());+ FieldStats fieldStats = searcher.fieldStats(query.getField());if (termUpTo > 0) {- return similarity.scorer(- boost, collectionStats, ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));+ return similarity.scorer(boost, fieldStats, ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));} else {return null; // no terms at all exist, we won't use similarity}diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.javaindex cc5f16bfe9e..76f120bed67 100644--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java@@ -33,10 +33,10 @@ import org.apache.lucene.queries.spans.SpanNearQuery;import org.apache.lucene.queries.spans.SpanOrQuery;import org.apache.lucene.queries.spans.SpanQuery;import org.apache.lucene.queries.spans.SpanTermQuery;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.WildcardQuery;import org.apache.lucene.search.similarities.ClassicSimilarity;@@ -367,13 +367,12 @@ public class TestPayloadScoreQuery extends LuceneTestCase {// idf used for phrase queries@Override- public Explanation idfExplain(- CollectionStatistics collectionStats, TermStatistics[] termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {return Explanation.match(1.0f, "Inexplicable");}@Override- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {return Explanation.match(1.0f, "Inexplicable");}}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.javaindex debd10ea83d..b595790a827 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java@@ -38,11 +38,11 @@ import org.apache.lucene.index.TermStates;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.BooleanQuery;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.DisiWrapper;import org.apache.lucene.search.DisjunctionDISIApproximation;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.Explanation;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Matches;import org.apache.lucene.search.Query;@@ -52,7 +52,7 @@ import org.apache.lucene.search.Scorer;import org.apache.lucene.search.ScorerSupplier;import org.apache.lucene.search.TermQuery;import org.apache.lucene.search.TermScorer;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.Weight;import org.apache.lucene.search.similarities.BM25Similarity;import org.apache.lucene.search.similarities.DFRSimilarity;@@ -72,8 +72,8 @@ import org.apache.lucene.util.SmallFloat;** <ol>* <li>Given a list of fields and weights, it pretends there is a synthetic combined field where- * all terms have been indexed. It computes new term and collection statistics for this- * combined field.+ * all terms have been indexed. It computes new term and field statistics for this combined+ * field.* <li>It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents.* </ol>*@@ -314,41 +314,37 @@ public final class CombinedFieldQuery extends Query implements Accountable {TermStates ts = TermStates.build(searcher, fieldTerms[i], true);termStates[i] = ts;if (ts.docFreq() > 0) {- TermStatistics termStats =- searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());+ TermStats termStats = searcher.termStats(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());docFreq = Math.max(termStats.docFreq(), docFreq);totalTermFreq += (double) field.weight * termStats.totalTermFreq();}}if (docFreq > 0) {- CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher);- TermStatistics pseudoTermStatistics =- new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));- this.simWeight =- searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics);+ FieldStats pseudoFieldStats = mergeFieldStats(searcher);+ TermStats pseudoTermStats =+ new TermStats(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));+ this.simWeight = searcher.getSimilarity().scorer(boost, pseudoFieldStats, pseudoTermStats);} else {this.simWeight = null;}}- private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher)- throws IOException {+ private FieldStats mergeFieldStats(IndexSearcher searcher) throws IOException {long maxDoc = 0;long docCount = 0;long sumTotalTermFreq = 0;long sumDocFreq = 0;for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {- CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field);- if (collectionStats != null) {- maxDoc = Math.max(collectionStats.maxDoc(), maxDoc);- docCount = Math.max(collectionStats.docCount(), docCount);- sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq);- sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq();+ FieldStats fieldStats = searcher.fieldStats(fieldWeight.field);+ if (fieldStats != null) {+ maxDoc = Math.max(fieldStats.maxDoc(), maxDoc);+ docCount = Math.max(fieldStats.docCount(), docCount);+ sumDocFreq = Math.max(fieldStats.sumDocFreq(), sumDocFreq);+ sumTotalTermFreq += (double) fieldWeight.weight * fieldStats.sumTotalTermFreq();}}- return new CollectionStatistics(- "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);+ return new FieldStats("pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);}@Overridediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.javaindex 218b2048f0f..9b86fba1125 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java@@ -51,7 +51,6 @@ import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.ScorerSupplier;import org.apache.lucene.search.SloppyPhraseMatcher;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;import org.apache.lucene.search.Weight;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.util.ArrayUtil;@@ -153,7 +152,7 @@ public class PhraseWildcardQuery extends Query {List<LeafReaderContext> sizeSortedSegments =new SegmentTermsSizeComparator().createTermsSizeSortedCopyOf(reader.leaves());- // TermsData will contain the collected TermState and TermStatistics for all the terms+ // TermsData will contain the collected TermState and TermStats for all the terms// of the phrase. It is filled during PhraseTerm.collectTermData() calls below.TermsData termsData = createTermsData(sizeSortedSegments.size());@@ -255,8 +254,8 @@ public class PhraseWildcardQuery extends Query {.getSimilarity().scorer(boost,- searcher.collectionStatistics(field),- termsData.termStatsList.toArray(new TermStatistics[0]));+ searcher.fieldStats(field),+ termsData.termStatsList.toArray(new org.apache.lucene.search.TermStats[0]));}@Override@@ -362,7 +361,8 @@ public class PhraseWildcardQuery extends Query {}/**- * Collects the {@link TermState} and {@link TermStatistics} for a single-term without expansion.+ * Collects the {@link TermState} and {@link org.apache.lucene.search.TermStats} for a single-term+ * without expansion.** @param termsData receives the collected data.*/@@ -406,13 +406,14 @@ public class PhraseWildcardQuery extends Query {// Collect the term stats across all segments.if (termStates.docFreq() > 0) {termsData.termStatsList.add(- searcher.termStatistics(term, termStates.docFreq(), termStates.totalTermFreq()));+ searcher.termStats(term, termStates.docFreq(), termStates.totalTermFreq()));}return numMatches;}/**- * Collects the {@link TermState} and {@link TermStatistics} for a multi-term with expansion.+ * Collects the {@link TermState} and {@link org.apache.lucene.search.TermStats} for a multi-term+ * with expansion.** @param remainingMultiTerms the number of remaining multi-terms to process, including the* current one, excluding the multi-terms already processed.@@ -473,8 +474,8 @@ public class PhraseWildcardQuery extends Query {}/**- * Collects the {@link TermState} list and {@link TermStatistics} for a multi-term on a specific- * index segment.+ * Collects the {@link TermState} list and {@link org.apache.lucene.search.TermStats} for a+ * multi-term on a specific index segment.** @param remainingExpansions the number of remaining expansions allowed for the segment.* @param shouldStopSegmentIteration to be set to true to stop the segment iteration calling this@@ -537,7 +538,8 @@ public class PhraseWildcardQuery extends Query {* Collect the term stats across all segments.** @param termStatsMap input map of already collected {@link TermStats}.- * @param termsData receives the {@link TermStatistics} computed for all {@link TermStats}.+ * @param termsData receives the {@link org.apache.lucene.search.TermStats} computed for all+ * {@link TermStats}.* @param termData receives all the collected {@link Term}.*/protected void collectMultiTermStats(@@ -555,7 +557,7 @@ public class PhraseWildcardQuery extends Query {TermStats termStats = termStatsEntry.getValue();if (termStats.docFreq > 0) {termsData.termStatsList.add(- searcher.termStatistics(term, termStats.docFreq, termStats.totalTermFreq));+ searcher.termStats(term, termStats.docFreq, termStats.totalTermFreq));}}}@@ -689,10 +691,11 @@ public class PhraseWildcardQuery extends Query {protected abstract Query getQuery();/**- * Collects {@link TermState} and {@link TermStatistics} for the term without expansion. It must- * be called only if {@link #hasExpansions()} returns false. Simplified version of {@code- * #collectTermData(PhraseWildcardQuery, IndexSearcher, List, int, int, TermsData)} with less- * arguments. This method throws {@link UnsupportedOperationException} if not overridden.+ * Collects {@link TermState} and {@link org.apache.lucene.search.TermStats} for the term+ * without expansion. It must be called only if {@link #hasExpansions()} returns false.+ * Simplified version of {@code #collectTermData(PhraseWildcardQuery, IndexSearcher, List, int,+ * int, TermsData)} with less arguments. This method throws {@link+ * UnsupportedOperationException} if not overridden.*/protected int collectTermData(PhraseWildcardQuery query,@@ -704,7 +707,8 @@ public class PhraseWildcardQuery extends Query {}/**- * Collects {@link TermState} and {@link TermStatistics} for the term (potentially expanded).+ * Collects {@link TermState} and {@link org.apache.lucene.search.TermStats} for the term+ * (potentially expanded).** @param termsData {@link TermsData} to update with the collected terms and stats.* @return The number of expansions or matches in all segments; or 0 if this term does not match@@ -842,14 +846,14 @@ public class PhraseWildcardQuery extends Query {}/**- * Holds the {@link TermState} and {@link TermStatistics} for all the matched and collected {@link- * Term}, for all phrase terms, for all segments.+ * Holds the {@link TermState} and {@link org.apache.lucene.search.TermStats} for all the matched+ * and collected {@link Term}, for all phrase terms, for all segments.*/protected static class TermsData {protected final int numTerms;protected final int numSegments;- protected final List<TermStatistics> termStatsList;+ protected final List<org.apache.lucene.search.TermStats> termStatsList;protected final TermData[] termDataPerPosition;protected int numTermsMatching;@@ -885,14 +889,14 @@ public class PhraseWildcardQuery extends Query {builder.append("numSegments=").append(numSegments);builder.append(", termDataPerPosition=").append(Arrays.asList(termDataPerPosition));builder.append(", termsStatsList=[");- for (TermStatistics termStatistics : termStatsList) {+ for (org.apache.lucene.search.TermStats termStats : termStatsList) {builder.append("{")- .append(termStatistics.term().utf8ToString())+ .append(termStats.term().utf8ToString()).append(", ")- .append(termStatistics.docFreq())+ .append(termStats.docFreq()).append(", ")- .append(termStatistics.totalTermFreq())+ .append(termStats.totalTermFreq()).append("}");}builder.append("]");diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.javaindex cbde5bf0ba1..f77478bed93 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java@@ -45,7 +45,7 @@ import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.Scorer;import org.apache.lucene.search.ScorerSupplier;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.Weight;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.util.Accountable;@@ -376,14 +376,13 @@ public class TermAutomatonQuery extends Query implements Accountable {this.automaton = automaton;this.termStates = termStates;this.similarity = searcher.getSimilarity();- List<TermStatistics> allTermStats = new ArrayList<>();+ List<TermStats> allTermStats = new ArrayList<>();for (IntObjectHashMap.IntObjectCursor<BytesRef> ent : idToTerm) {if (ent.value != null) {TermStates ts = termStates.get(ent.key);if (ts.docFreq() > 0) {allTermStats.add(- searcher.termStatistics(- new Term(field, ent.value), ts.docFreq(), ts.totalTermFreq()));+ searcher.termStats(new Term(field, ent.value), ts.docFreq(), ts.totalTermFreq()));}}}@@ -394,8 +393,8 @@ public class TermAutomatonQuery extends Query implements Accountable {stats =similarity.scorer(boost,- searcher.collectionStatistics(field),- allTermStats.toArray(new TermStatistics[allTermStats.size()]));+ searcher.fieldStats(field),+ allTermStats.toArray(new TermStats[allTermStats.size()]));}}diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.javaindex 7d8f9a5bdec..aff1df264d7 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java@@ -31,13 +31,13 @@ import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.MultiReader;import org.apache.lucene.index.Term;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.CollectorManager;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.MatchNoDocsQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.TopScoreDocCollector;import org.apache.lucene.search.TopScoreDocCollectorManager;@@ -586,13 +586,12 @@ public class TestCombinedFieldQuery extends LuceneTestCase {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {- return new BM25Similarity().scorer(boost, collectionStats, termStats);+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {+ return new BM25Similarity().scorer(boost, fieldStats, termStats);}}- public void testOverrideCollectionStatistics() throws IOException {+ public void testOverrideFieldStats() throws IOException {Directory dir = newDirectory();IndexWriterConfig iwc = new IndexWriterConfig();Similarity similarity = randomCompatibleSimilarity();@@ -639,8 +638,8 @@ public class TestCombinedFieldQuery extends LuceneTestCase {IndexSearcher searcher =new IndexSearcher(reader) {@Override- public CollectionStatistics collectionStatistics(String field) throws IOException {- CollectionStatistics shardStatistics = super.collectionStatistics(field);+ public FieldStats fieldStats(String field) throws IOException {+ FieldStats shardStatistics = super.fieldStats(field);int extraSumTotalTermFreq;if (field.equals("a")) {extraSumTotalTermFreq = extraSumTotalTermFreqA;@@ -651,7 +650,7 @@ public class TestCombinedFieldQuery extends LuceneTestCase {} else {throw new AssertionError("should never be called");}- return new CollectionStatistics(+ return new FieldStats(field,shardStatistics.maxDoc() + extraMaxDoc,shardStatistics.docCount() + extraDocCount,diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.javaindex 5c3f97db33c..a0d6c624ef4 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java@@ -40,9 +40,9 @@ import org.apache.lucene.index.MultiDocValues;import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.Term;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -634,8 +634,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {throw new UnsupportedOperationException();}}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.javaindex 7a3785d2b59..585cb715664 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java@@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.Term;import org.apache.lucene.index.TermStates;-import org.apache.lucene.search.CollectionStatistics;+import org.apache.lucene.search.FieldStats;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.QueryVisitor;@@ -37,7 +37,7 @@ import org.apache.lucene.search.ScoreDoc;import org.apache.lucene.search.SearcherLifetimeManager;import org.apache.lucene.search.SearcherManager;import org.apache.lucene.search.Sort;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.TopFieldDocs;import org.apache.lucene.store.Directory;@@ -101,7 +101,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {}}- // We share collection stats for these fields on each node+ // We share field stats for these fields on each node// reopen:private final String[] fieldsToShare = new String[] {"body", "title"};@@ -109,7 +109,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {// other nodes. This is just a mock (since it goes and// directly updates all other nodes, in RAM)... in a real// env this would hit the wire, sending version &- // collection stats to all other nodes:+ // field stats to all other nodes:void broadcastNodeReopen(int nodeID, long version, IndexSearcher newSearcher) throws IOException {if (VERBOSE) {@@ -122,16 +122,16 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {+ newSearcher.getIndexReader().maxDoc());}- // Broadcast new collection stats for this node to all+ // Broadcast new field stats for this node to all// other nodes:for (String field : fieldsToShare) {- final CollectionStatistics stats = newSearcher.collectionStatistics(field);+ final FieldStats stats = newSearcher.fieldStats(field);if (stats != null) {for (NodeState node : nodes) {- // Don't put my own collection stats into the cache;+ // Don't put my own field stats into the cache;// we pull locally:if (node.myNodeID != nodeID) {- node.collectionStatsCache.put(new FieldAndShardVersion(nodeID, version, field), stats);+ node.fieldStatsCache.put(new FieldAndShardVersion(nodeID, version, field), stats);}}}@@ -169,10 +169,10 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {// Mock: in a real env, this would hit the wire and get// term stats from remote node- Map<Term, TermStatistics> getNodeTermStats(Set<Term> terms, int nodeID, long version)+ Map<Term, TermStats> getNodeTermStats(Set<Term> terms, int nodeID, long version)throws IOException {final NodeState node = nodes[nodeID];- final Map<Term, TermStatistics> stats = new HashMap<>();+ final Map<Term, TermStats> stats = new HashMap<>();final IndexSearcher s = node.searchers.acquire(version);if (s == null) {throw new SearcherExpiredException("node=" + nodeID + " version=" + version);@@ -181,7 +181,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {for (Term term : terms) {final TermStates ts = TermStates.build(s, term, true);if (ts.docFreq() > 0) {- stats.put(term, s.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));+ stats.put(term, s.termStats(term, ts.docFreq(), ts.totalTermFreq()));}}} finally {@@ -204,10 +204,8 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {// local cache...? And still LRU otherwise (for the// still-live searchers).- private final Map<FieldAndShardVersion, CollectionStatistics> collectionStatsCache =- new ConcurrentHashMap<>();- private final Map<TermAndShardVersion, TermStatistics> termStatsCache =- new ConcurrentHashMap<>();+ private final Map<FieldAndShardVersion, FieldStats> fieldStatsCache = new ConcurrentHashMap<>();+ private final Map<TermAndShardVersion, TermStats> termStatsCache = new ConcurrentHashMap<>();/*** Matches docs in the local shard but scores based on aggregated stats ("mock distributed@@ -249,7 +247,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {}}if (missing.size() != 0) {- for (Map.Entry<Term, TermStatistics> ent :+ for (Map.Entry<Term, TermStats> ent :getNodeTermStats(missing, nodeID, nodeVersions[nodeID]).entrySet()) {if (ent.getValue() != null) {final TermAndShardVersion key =@@ -264,16 +262,15 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {}@Override- public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)- throws IOException {+ public TermStats termStats(Term term, int docFreq, long totalTermFreq) throws IOException {assert term != null;long distributedDocFreq = 0;long distributedTotalTermFreq = 0;for (int nodeID = 0; nodeID < nodeVersions.length; nodeID++) {- final TermStatistics subStats;+ final TermStats subStats;if (nodeID == myNodeID) {- subStats = super.termStatistics(term, docFreq, totalTermFreq);+ subStats = super.termStats(term, docFreq, totalTermFreq);} else {final TermAndShardVersion key =new TermAndShardVersion(nodeID, nodeVersions[nodeID], term);@@ -290,11 +287,11 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {distributedTotalTermFreq += nodeTotalTermFreq;}assert distributedDocFreq > 0;- return new TermStatistics(term.bytes(), distributedDocFreq, distributedTotalTermFreq);+ return new TermStats(term.bytes(), distributedDocFreq, distributedTotalTermFreq);}@Override- public CollectionStatistics collectionStatistics(String field) throws IOException {+ public FieldStats fieldStats(String field) throws IOException {// TODO: we could compute this on init and cache,// since we are re-inited whenever any nodes have a// new reader@@ -306,11 +303,11 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {for (int nodeID = 0; nodeID < nodeVersions.length; nodeID++) {final FieldAndShardVersion key =new FieldAndShardVersion(nodeID, nodeVersions[nodeID], field);- final CollectionStatistics nodeStats;+ final FieldStats nodeStats;if (nodeID == myNodeID) {- nodeStats = super.collectionStatistics(field);+ nodeStats = super.fieldStats(field);} else {- nodeStats = collectionStatsCache.get(key);+ nodeStats = fieldStatsCache.get(key);}if (nodeStats == null) {continue; // field not in sub at all@@ -332,7 +329,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {if (maxDoc == 0) {return null; // field not found across any node whatsoever} else {- return new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);+ return new FieldStats(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);}}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.javaindex 8943c8f7a89..fe08c39f8c4 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java@@ -17,9 +17,9 @@package org.apache.lucene.tests.search.similarities;import org.apache.lucene.index.FieldInvertState;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.Similarity;/** wraps a similarity with checks for testing */@@ -48,16 +48,15 @@ public class AssertingSimilarity extends Similarity {}@Override- public SimScorer scorer(- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {assert boost >= 0;- assert collectionStats != null;+ assert fieldStats != null;assert termStats.length > 0;- for (TermStatistics term : termStats) {+ for (TermStats term : termStats) {assert term != null;}// TODO: check that TermStats is in bounds with respect to collection? e.g. docFreq <= maxDoc- SimScorer scorer = delegate.scorer(boost, collectionStats, termStats);+ SimScorer scorer = delegate.scorer(boost, fieldStats, termStats);assert scorer != null;return new AssertingSimScorer(scorer, boost);}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.javaindex c56a6417384..4dbfa9f7910 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java@@ -22,9 +22,9 @@ import org.apache.lucene.document.Document;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.TextField;import org.apache.lucene.index.LeafReader;-import org.apache.lucene.search.CollectionStatistics;import org.apache.lucene.search.Explanation;-import org.apache.lucene.search.TermStatistics;+import org.apache.lucene.search.FieldStats;+import org.apache.lucene.search.TermStats;import org.apache.lucene.search.similarities.IndriDirichletSimilarity;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.search.similarities.Similarity.BulkSimScorer;@@ -83,7 +83,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {/*** returns a random corpus that is at least possible given the norm value for a single document.*/- static CollectionStatistics newCorpus(Random random, int norm) {+ static FieldStats newCorpus(Random random, int norm) {// lower bound of tokens in the collection (you produced this norm somehow)final int lowerBound;if (norm == 0) {@@ -177,13 +177,13 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {sumTotalTermFreq = TestUtil.nextLong(random, sumDocFreq - 1 + lowerBound, upperBound);break;}- return new CollectionStatistics("field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);+ return new FieldStats("field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);}private static final BytesRef TERM = new BytesRef("term");/** returns new random term, that fits within the bounds of the corpus */- static TermStatistics newTerm(Random random, CollectionStatistics corpus) {+ static TermStats newTerm(Random random, FieldStats corpus) {final long docFreq;switch (random.nextInt(3)) {case 0:@@ -227,7 +227,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {break;}}- return new TermStatistics(TERM, docFreq, totalTermFreq);+ return new TermStats(TERM, docFreq, totalTermFreq);}/**@@ -262,9 +262,9 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {for (int j = 0; j < 3; j++) {// for each norm value...for (int k = 1; k < 256; k++) {- CollectionStatistics corpus = newCorpus(random, k);+ FieldStats corpus = newCorpus(random, k);for (int l = 0; l < 10; l++) {- TermStatistics term = newTerm(random, corpus);+ TermStats term = newTerm(random, corpus);final float freq;if (term.totalTermFreq() == term.docFreq()) {// omit TF@@ -356,12 +356,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {* for that scenario*/private static void doTestScoring(- Similarity similarity,- CollectionStatistics corpus,- TermStatistics term,- float boost,- float freq,- int norm)+ Similarity similarity, FieldStats corpus, TermStats term, float boost, float freq, int norm)throws IOException {boolean success = false;SimScorer scorer = similarity.scorer(boost, corpus, term);@@ -463,8 +458,8 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {// check score(term-1), given the same freq/norm it should be >= score(term) [scores// non-decreasing as terms get rarer]if (term.docFreq() > 1 && freq < term.totalTermFreq()) {- TermStatistics prevTerm =- new TermStatistics(term.term(), term.docFreq() - 1, term.totalTermFreq() - 1);+ TermStats prevTerm =+ new TermStats(term.term(), term.docFreq() - 1, term.totalTermFreq() - 1);SimScorer prevTermScorer = similarity.scorer(boost, corpus, term);float prevTermScore = prevTermScorer.score(freq, norm);// check that score isn't infinite or negative@@ -523,8 +518,8 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {public void testBulkScore() throws IOException {Random random = random();Similarity similarity = getSimilarity(random);- CollectionStatistics corpus = newCorpus(random, 1);- TermStatistics term = newTerm(random, corpus);+ FieldStats corpus = newCorpus(random, 1);+ TermStats term = newTerm(random, corpus);SimScorer scorer = similarity.scorer(random().nextFloat(5f), corpus, term);BulkSimScorer bulkScorer = scorer.asBulkSimScorer();int freqUpperBound =