Apache Lucene open-source search software
Move `CombinedFieldQuery` to the core module. (#14236)
Adrien Grand239059e69cf4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex cd39edc3227..761fafeba09 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -52,6 +52,9 @@ API Changes* GITHUB#14209: Deprecate Operations.union(Automaton,Automaton) andconcatenate(Automaton,Automaton) in favor of the methods taking List. (Robert Muir)+* GITHUB#14236: CombinedFieldQuery moved from lucene-sandbox to lucene-core.+ (Adrien Grand)+New Features---------------------diff --git a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.javanew file mode 100644index 00000000000..42659e3b37e--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java@@ -0,0 +1,462 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Objects;+import java.util.TreeMap;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.TermState;+import org.apache.lucene.index.TermStates;+import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.search.similarities.BM25Similarity;+import org.apache.lucene.search.similarities.DFRSimilarity;+import org.apache.lucene.search.similarities.Similarity;+import org.apache.lucene.util.Accountable;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOSupplier;+import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.SmallFloat;++/**+ * A {@link Query} that treats multiple fields as a single stream and scores terms as if they had+ * been indexed in a single field whose values would be the union of the values of the provided+ * fields.+ *+ * <p>The query works as follows:+ *+ * <ol>+ * <li>Given a list of fields and weights, it pretends there is a synthetic combined field where+ * all terms have been indexed. It computes new term and collection statistics for this+ * combined field.+ * <li>It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents.+ * </ol>+ *+ * <p>In order for a similarity to be compatible, {@link Similarity#computeNorm} must be additive:+ * the norm of the combined field is the sum of norms for each individual field. The norms must also+ * be encoded using {@link SmallFloat#intToByte4}. These requirements hold for all similarities that+ * don't customize {@link Similarity#computeNorm}, which includes {@link BM25Similarity} and {@link+ * DFRSimilarity}. Per-field similarities are not supported.+ *+ * <p>The query also requires that either all fields or no fields have norms enabled. Having only+ * some fields with norms enabled can result in errors.+ *+ * <p>This query assumes that all fields share the same analyzer. Scores may not make much sense if+ * all fields don't have the same analyzer.+ *+ * <p>The scoring is based on BM25F's simple formula described in:+ * http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf. This query implements the+ * same approach but allows other similarities besides {@link+ * org.apache.lucene.search.similarities.BM25Similarity}.+ *+ * @lucene.experimental+ */+public final class CombinedFieldQuery extends Query implements Accountable {+ private static final long BASE_RAM_BYTES =+ RamUsageEstimator.shallowSizeOfInstance(CombinedFieldQuery.class);++ /** A builder for {@link CombinedFieldQuery}. */+ public static class Builder {+ private final Map<String, FieldAndWeight> fieldAndWeights = new HashMap<>();+ private final BytesRef term;++ /** Create a builder for the given term {@link String}. */+ public Builder(String term) {+ this.term = new BytesRef(term);+ }++ /** Create a builder for the given term bytes. */+ public Builder(BytesRef term) {+ this.term = BytesRef.deepCopyOf(term);+ }++ /**+ * Adds a field to this builder.+ *+ * @param field The field name.+ */+ public Builder addField(String field) {+ return addField(field, 1f);+ }++ /**+ * Adds a field to this builder.+ *+ * @param field The field name.+ * @param weight The weight associated to this field.+ */+ public Builder addField(String field, float weight) {+ if (weight < 1) {+ throw new IllegalArgumentException("weight must be greater or equal to 1");+ }+ fieldAndWeights.put(field, new FieldAndWeight(field, weight));+ return this;+ }++ /** Builds the {@link CombinedFieldQuery}. */+ public CombinedFieldQuery build() {+ if (fieldAndWeights.size() > IndexSearcher.getMaxClauseCount()) {+ throw new IndexSearcher.TooManyClauses();+ }+ return new CombinedFieldQuery(new TreeMap<>(fieldAndWeights), term);+ }+ }++ record FieldAndWeight(String field, float weight) {}++ // sorted map for fields.+ private final TreeMap<String, FieldAndWeight> fieldAndWeights;+ // term bytes+ private final BytesRef term;+ // array of terms per field, sorted by field+ private final Term[] fieldTerms;++ private final long ramBytesUsed;++ private CombinedFieldQuery(TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef term) {+ this.fieldAndWeights = fieldAndWeights;+ this.term = Objects.requireNonNull(term);+ if (fieldAndWeights.size() > IndexSearcher.getMaxClauseCount()) {+ throw new IndexSearcher.TooManyClauses();+ }+ this.fieldTerms = new Term[fieldAndWeights.size()];+ int pos = 0;+ for (String field : fieldAndWeights.keySet()) {+ fieldTerms[pos++] = new Term(field, term);+ }++ this.ramBytesUsed =+ BASE_RAM_BYTES+ + RamUsageEstimator.sizeOfObject(fieldAndWeights)+ + RamUsageEstimator.sizeOfObject(fieldTerms)+ + RamUsageEstimator.sizeOfObject(term);+ }++ @Override+ public String toString(String field) {+ StringBuilder builder = new StringBuilder("CombinedFieldQuery((");+ int pos = 0;+ for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {+ if (pos++ != 0) {+ builder.append(" ");+ }+ builder.append(fieldWeight.field);+ if (fieldWeight.weight != 1f) {+ builder.append("^");+ builder.append(fieldWeight.weight);+ }+ }+ builder.append(")(");+ builder.append(Term.toString(term));+ builder.append("))");+ return builder.toString();+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (sameClassAs(o) == false) return false;+ CombinedFieldQuery that = (CombinedFieldQuery) o;+ return Objects.equals(fieldAndWeights, that.fieldAndWeights) && term.equals(that.term);+ }++ @Override+ public int hashCode() {+ int result = classHash();+ result = 31 * result + Objects.hash(fieldAndWeights);+ result = 31 * result + term.hashCode();+ return result;+ }++ @Override+ public long ramBytesUsed() {+ return ramBytesUsed;+ }++ @Override+ public Query rewrite(IndexSearcher indexSearcher) throws IOException {+ if (fieldAndWeights.isEmpty()) {+ return new BooleanQuery.Builder().build();+ }+ return this;+ }++ @Override+ public void visit(QueryVisitor visitor) {+ Term[] selectedTerms =+ Arrays.stream(fieldTerms).filter(t -> visitor.acceptField(t.field())).toArray(Term[]::new);+ if (selectedTerms.length > 0) {+ QueryVisitor v = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this);+ v.consumeTerms(this, selectedTerms);+ }+ }++ private BooleanQuery rewriteToBoolean() {+ // rewrite to a simple disjunction if the score is not needed.+ BooleanQuery.Builder bq = new BooleanQuery.Builder();+ for (Term term : fieldTerms) {+ bq.add(new TermQuery(term), BooleanClause.Occur.SHOULD);+ }+ return bq.build();+ }++ @Override+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)+ throws IOException {+ validateConsistentNorms(searcher.getIndexReader());+ if (scoreMode.needsScores()) {+ return new CombinedFieldWeight(this, searcher, scoreMode, boost);+ } else {+ // rewrite to a simple disjunction if the score is not needed.+ Query bq = rewriteToBoolean();+ return searcher.rewrite(bq).createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);+ }+ }++ private void validateConsistentNorms(IndexReader reader) {+ boolean allFieldsHaveNorms = true;+ boolean noFieldsHaveNorms = true;++ for (LeafReaderContext context : reader.leaves()) {+ FieldInfos fieldInfos = context.reader().getFieldInfos();+ for (String field : fieldAndWeights.keySet()) {+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);+ if (fieldInfo != null) {+ allFieldsHaveNorms &= fieldInfo.hasNorms();+ noFieldsHaveNorms &= fieldInfo.omitsNorms();+ }+ }+ }++ if (allFieldsHaveNorms == false && noFieldsHaveNorms == false) {+ throw new IllegalArgumentException(+ getClass().getSimpleName()+ + " requires norms to be consistent across fields: some fields cannot "+ + " have norms enabled, while others have norms disabled");+ }+ }++ class CombinedFieldWeight extends Weight {+ private final IndexSearcher searcher;+ private final TermStates[] termStates;+ private final Similarity.SimScorer simWeight;++ CombinedFieldWeight(Query query, IndexSearcher searcher, ScoreMode scoreMode, float boost)+ throws IOException {+ super(query);+ assert scoreMode.needsScores();+ this.searcher = searcher;+ long docFreq = 0;+ long totalTermFreq = 0;+ termStates = new TermStates[fieldTerms.length];+ for (int i = 0; i < termStates.length; i++) {+ FieldAndWeight field = fieldAndWeights.get(fieldTerms[i].field());+ TermStates ts = TermStates.build(searcher, fieldTerms[i], true);+ termStates[i] = ts;+ if (ts.docFreq() > 0) {+ TermStatistics termStats =+ searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());+ docFreq = Math.max(termStats.docFreq(), docFreq);+ totalTermFreq += (double) field.weight * termStats.totalTermFreq();+ }+ }+ if (docFreq > 0) {+ CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher);+ TermStatistics pseudoTermStatistics =+ new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));+ this.simWeight =+ searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics);+ } else {+ this.simWeight = null;+ }+ }++ private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher)+ throws IOException {+ long maxDoc = 0;+ long docCount = 0;+ long sumTotalTermFreq = 0;+ long sumDocFreq = 0;+ for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {+ CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field);+ if (collectionStats != null) {+ maxDoc = Math.max(collectionStats.maxDoc(), maxDoc);+ docCount = Math.max(collectionStats.docCount(), docCount);+ sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq);+ sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq();+ }+ }++ return new CollectionStatistics(+ "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);+ }++ @Override+ public Matches matches(LeafReaderContext context, int doc) throws IOException {+ Weight weight =+ searcher.rewrite(rewriteToBoolean()).createWeight(searcher, ScoreMode.COMPLETE, 1f);+ return weight.matches(context, doc);+ }++ @Override+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {+ Scorer scorer = scorer(context);+ if (scorer != null) {+ int newDoc = scorer.iterator().advance(doc);+ if (newDoc == doc) {+ assert scorer instanceof CombinedFieldScorer;+ float freq = ((CombinedFieldScorer) scorer).freq();+ MultiNormsLeafSimScorer docScorer =+ new MultiNormsLeafSimScorer(+ simWeight, context.reader(), fieldAndWeights.values(), true);+ Explanation freqExplanation = Explanation.match(freq, "termFreq=" + freq);+ Explanation scoreExplanation = docScorer.explain(doc, freqExplanation);+ return Explanation.match(+ scoreExplanation.getValue(),+ "weight(" + getQuery() + " in " + doc + "), result of:",+ scoreExplanation);+ }+ }+ return Explanation.noMatch("no matching term");+ }++ @Override+ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {+ List<PostingsEnum> iterators = new ArrayList<>();+ List<FieldAndWeight> fields = new ArrayList<>();+ long cost = 0;+ for (int i = 0; i < fieldTerms.length; i++) {+ IOSupplier<TermState> supplier = termStates[i].get(context);+ TermState state = supplier == null ? null : supplier.get();+ if (state != null) {+ TermsEnum termsEnum = context.reader().terms(fieldTerms[i].field()).iterator();+ termsEnum.seekExact(fieldTerms[i].bytes(), state);+ PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.FREQS);+ iterators.add(postingsEnum);+ fields.add(fieldAndWeights.get(fieldTerms[i].field()));+ cost += postingsEnum.cost();+ }+ }++ if (iterators.isEmpty()) {+ return null;+ }++ MultiNormsLeafSimScorer scoringSimScorer =+ new MultiNormsLeafSimScorer(simWeight, context.reader(), fieldAndWeights.values(), true);++ final long finalCost = cost;+ return new ScorerSupplier() {++ @Override+ public Scorer get(long leadCost) throws IOException {+ // we use termscorers + disjunction as an impl detail+ List<DisiWrapper> wrappers = new ArrayList<>(iterators.size());+ for (int i = 0; i < iterators.size(); i++) {+ float weight = fields.get(i).weight;+ wrappers.add(+ new WeightedDisiWrapper(new TermScorer(iterators.get(i), simWeight, null), weight));+ }+ // Even though it is called approximation, it is accurate since none of+ // the sub iterators are two-phase iterators.+ DisjunctionDISIApproximation iterator =+ new DisjunctionDISIApproximation(wrappers, leadCost);+ return new CombinedFieldScorer(iterator, scoringSimScorer);+ }++ @Override+ public long cost() {+ return finalCost;+ }+ };+ }++ @Override+ public boolean isCacheable(LeafReaderContext ctx) {+ return false;+ }+ }++ private static class WeightedDisiWrapper extends DisiWrapper {+ final PostingsEnum postingsEnum;+ final float weight;++ WeightedDisiWrapper(Scorer scorer, float weight) {+ super(scorer, false);+ this.weight = weight;+ this.postingsEnum = (PostingsEnum) scorer.iterator();+ }++ float freq() throws IOException {+ return weight * postingsEnum.freq();+ }+ }++ private static class CombinedFieldScorer extends Scorer {+ private final DisjunctionDISIApproximation iterator;+ private final MultiNormsLeafSimScorer simScorer;+ private final float maxScore;++ CombinedFieldScorer(DisjunctionDISIApproximation iterator, MultiNormsLeafSimScorer simScorer) {+ this.iterator = iterator;+ this.simScorer = simScorer;+ this.maxScore = simScorer.getSimScorer().score(Float.POSITIVE_INFINITY, 1L);+ }++ @Override+ public int docID() {+ return iterator.docID();+ }++ float freq() throws IOException {+ DisiWrapper w = iterator.topList();+ float freq = ((WeightedDisiWrapper) w).freq();+ for (w = w.next; w != null; w = w.next) {+ freq += ((WeightedDisiWrapper) w).freq();+ if (freq < 0) { // overflow+ return Integer.MAX_VALUE;+ }+ }+ return freq;+ }++ @Override+ public float score() throws IOException {+ return simScorer.score(iterator.docID(), freq());+ }++ @Override+ public DocIdSetIterator iterator() {+ return iterator;+ }++ @Override+ public float getMaxScore(int upTo) throws IOException {+ return maxScore;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.java b/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.javanew file mode 100644index 00000000000..c6720b362d9--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.java@@ -0,0 +1,177 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.HashSet;+import java.util.List;+import java.util.Objects;+import java.util.Set;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.search.CombinedFieldQuery.FieldAndWeight;+import org.apache.lucene.search.similarities.Similarity.SimScorer;+import org.apache.lucene.util.SmallFloat;++/**+ * Scorer that sums document's norms from multiple fields.+ *+ * <p>For all fields, norms must be encoded using {@link SmallFloat#intToByte4}. This scorer also+ * requires that either all fields or no fields have norms enabled. Having only some fields with+ * norms enabled can result in errors or undefined behavior.+ */+final class MultiNormsLeafSimScorer {+ /** Cache of decoded norms. */+ private static final float[] LENGTH_TABLE = new float[256];++ static {+ for (int i = 0; i < 256; i++) {+ LENGTH_TABLE[i] = SmallFloat.byte4ToInt((byte) i);+ }+ }++ private final SimScorer scorer;+ private final NumericDocValues norms;++ /** Sole constructor: Score documents of {@code reader} with {@code scorer}. */+ MultiNormsLeafSimScorer(+ SimScorer scorer,+ LeafReader reader,+ Collection<FieldAndWeight> normFields,+ boolean needsScores)+ throws IOException {+ this.scorer = Objects.requireNonNull(scorer);+ if (needsScores) {+ final List<NumericDocValues> normsList = new ArrayList<>();+ final List<Float> weightList = new ArrayList<>();+ final Set<String> duplicateCheckingSet = new HashSet<>();+ for (FieldAndWeight field : normFields) {+ assert duplicateCheckingSet.add(field.field())+ : "There is a duplicated field ["+ + field.field()+ + "] used to construct MultiNormsLeafSimScorer";++ NumericDocValues norms = reader.getNormValues(field.field());+ if (norms != null) {+ normsList.add(norms);+ weightList.add(field.weight());+ }+ }++ if (normsList.isEmpty()) {+ norms = null;+ } else {+ final NumericDocValues[] normsArr = normsList.toArray(new NumericDocValues[0]);+ final float[] weightArr = new float[normsList.size()];+ for (int i = 0; i < weightList.size(); i++) {+ weightArr[i] = weightList.get(i);+ }+ norms = new MultiFieldNormValues(normsArr, weightArr);+ }+ } else {+ norms = null;+ }+ }++ SimScorer getSimScorer() {+ return scorer;+ }++ private long getNormValue(int doc) throws IOException {+ if (norms != null) {+ boolean found = norms.advanceExact(doc);+ assert found;+ return norms.longValue();+ } else {+ return 1L; // default norm+ }+ }++ /**+ * Score the provided document assuming the given term document frequency. This method must be+ * called on non-decreasing sequences of doc ids.+ *+ * @see SimScorer#score(float, long)+ */+ public float score(int doc, float freq) throws IOException {+ return scorer.score(freq, getNormValue(doc));+ }++ /**+ * Explain the score for the provided document assuming the given term document frequency. This+ * method must be called on non-decreasing sequences of doc ids.+ *+ * @see SimScorer#explain(Explanation, long)+ */+ public Explanation explain(int doc, Explanation freqExpl) throws IOException {+ return scorer.explain(freqExpl, getNormValue(doc));+ }++ private static class MultiFieldNormValues extends NumericDocValues {+ private final NumericDocValues[] normsArr;+ private final float[] weightArr;+ private long current;+ private int docID = -1;++ MultiFieldNormValues(NumericDocValues[] normsArr, float[] weightArr) {+ this.normsArr = normsArr;+ this.weightArr = weightArr;+ }++ @Override+ public long longValue() {+ return current;+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ float normValue = 0;+ boolean found = false;+ for (int i = 0; i < normsArr.length; i++) {+ if (normsArr[i].advanceExact(target)) {+ normValue +=+ weightArr[i] * LENGTH_TABLE[Byte.toUnsignedInt((byte) normsArr[i].longValue())];+ found = true;+ }+ }+ current = SmallFloat.intToByte4(Math.round(normValue));+ return found;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int nextDoc() {+ throw new UnsupportedOperationException();+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException();+ }++ @Override+ public long cost() {+ throw new UnsupportedOperationException();+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/package-info.java b/lucene/core/src/java/org/apache/lucene/search/package-info.javaindex 98b60d94222..664e8e61693 100644--- a/lucene/core/src/java/org/apache/lucene/search/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/search/package-info.java@@ -278,6 +278,30 @@* <p>See the {@link org.apache.lucene.search.similarities} package documentation for information on* the built-in available scoring models and extending or changing Similarity.*+ * <h3>Scoring multiple fields</h3>+ *+ * <p>In the real world, documents often have multiple fields with different degrees of relevance. A+ * robust way of scoring across multiple fields is called BM25F, which is implemented via {@link+ * org.apache.lucene.search.CombinedFieldQuery}. It scores documents with multiple fields as if+ * their content had been indexed in a single combined field. It supports configuring per-field+ * boosts where the value of the boost is interpreted as the number of times that the content of the+ * field exists in the virtual combined field.+ *+ * <p>Here is an example that constructs a query on "apache OR lucene" on fields "title" with a+ * boost of 10, and "body" with a boost of 1:+ *+ * <pre class="prettyprint">+ * BooleanQuery.Builder builder = new BooleanQuery.Builder();+ * for (String term : new String[] { "apache", "lucene" }) {+ * Query query = new CombinedFieldQuery(term)+ * .addField("title", 10f)+ * .addField("body", 1f)+ * .build();+ * builder.add(query, Occur.SHOULD);+ * }+ * Query query = builder.build();+ * </pre>+ ** <h3>Integrating field values into the score</h3>** <p>While similarities help score a document relatively to a query, it is also common fordiff --git a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.javanew file mode 100644index 00000000000..ed0c051d0ba--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java@@ -0,0 +1,563 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;++import com.carrotsearch.randomizedtesting.generators.RandomPicks;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field.Store;+import org.apache.lucene.document.StringField;+import org.apache.lucene.document.TextField;+import org.apache.lucene.index.FieldInvertState;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.MultiReader;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.similarities.BM25Similarity;+import org.apache.lucene.search.similarities.BooleanSimilarity;+import org.apache.lucene.search.similarities.ClassicSimilarity;+import org.apache.lucene.search.similarities.LMDirichletSimilarity;+import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity;+import org.apache.lucene.search.similarities.Similarity;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.MMapDirectory;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.search.CheckHits;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestCombinedFieldQuery extends LuceneTestCase {+ public void testInvalid() {+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("foo");+ IllegalArgumentException exc =+ expectThrows(IllegalArgumentException.class, () -> builder.addField("foo", 0.5f));+ assertEquals(exc.getMessage(), "weight must be greater or equal to 1");+ }++ public void testRewrite() throws IOException {+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("foo");+ IndexReader reader = new MultiReader();+ IndexSearcher searcher = new IndexSearcher(reader);+ Query actual = searcher.rewrite(builder.build());+ assertEquals(new MatchNoDocsQuery(), actual);+ builder.addField("field", 1f);+ Query query = builder.build();+ actual = searcher.rewrite(builder.build());+ assertEquals(query, actual);+ }++ public void testEqualsAndHashCode() {+ CombinedFieldQuery query1 =+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2").build();++ CombinedFieldQuery query2 =+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2", 1.3f).build();+ assertNotEquals(query1, query2);+ assertNotEquals(query1.hashCode(), query2.hashCode());++ CombinedFieldQuery query3 =+ new CombinedFieldQuery.Builder("value").addField("field3").addField("field4").build();+ assertNotEquals(query1, query3);+ assertNotEquals(query1.hashCode(), query2.hashCode());++ CombinedFieldQuery duplicateQuery1 =+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2").build();+ assertEquals(query1, duplicateQuery1);+ assertEquals(query1.hashCode(), duplicateQuery1.hashCode());+ }++ public void testToString() {+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("bar");+ assertEquals("CombinedFieldQuery(()(bar))", builder.build().toString());+ builder.addField("foo", 1f);+ assertEquals("CombinedFieldQuery((foo)(bar))", builder.build().toString());+ builder.addField("title", 3f);+ assertEquals("CombinedFieldQuery((foo title^3.0)(bar))", builder.build().toString());+ }++ public void testSameScore() throws IOException {+ Directory dir = newDirectory();+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ Document doc = new Document();+ doc.add(new StringField("f", "a", Store.NO));+ w.addDocument(doc);++ doc = new Document();+ doc.add(new StringField("g", "a", Store.NO));+ for (int i = 0; i < 10; ++i) {+ w.addDocument(doc);+ }++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(similarity);+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("a").addField("f", 1f).addField("g", 1f).build();+ TopScoreDocCollectorManager collectorManager =+ new TopScoreDocCollectorManager(+ Math.min(reader.numDocs(), Integer.MAX_VALUE), Integer.MAX_VALUE);+ TopDocs topDocs = searcher.search(query, collectorManager);+ assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits);+ // All docs must have the same score+ for (int i = 0; i < topDocs.scoreDocs.length; ++i) {+ assertEquals(topDocs.scoreDocs[0].score, topDocs.scoreDocs[i].score, 0.0f);+ }++ reader.close();+ w.close();+ dir.close();+ }++ public void testScoringWithMultipleFieldTermsMatch() throws IOException {+ int numMatchDoc = randomIntBetween(100, 500);+ int numHits = randomIntBetween(1, 100);+ int boost1 = Math.max(1, random().nextInt(5));+ int boost2 = Math.max(1, random().nextInt(5));++ Directory dir = newDirectory();+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ // adding potentially matching doc+ for (int i = 0; i < numMatchDoc; i++) {+ Document doc = new Document();++ int freqA = random().nextInt(20) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo", Store.NO));+ }++ freqA = random().nextInt(20) + 1;+ if (randomBoolean()) {+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo" + j, Store.NO));+ }+ }++ freqA = random().nextInt(20) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "zoo", Store.NO));+ }++ int freqB = random().nextInt(20) + 1;+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "zoo", Store.NO));+ }++ freqB = random().nextInt(20) + 1;+ if (randomBoolean()) {+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "zoo" + j, Store.NO));+ }+ }++ int freqC = random().nextInt(20) + 1;+ for (int j = 0; j < freqC; j++) {+ doc.add(new TextField("c", "bla" + j, Store.NO));+ }+ w.addDocument(doc);+ }++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(similarity);++ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("foo")+ .addField("a", (float) boost1)+ .addField("b", (float) boost2)+ .build();++ CollectorManager<TopScoreDocCollector, TopDocs> completeManager =+ new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);++ searcher.search(query, completeManager);++ reader.close();+ w.close();+ dir.close();+ }++ public void testNormsDisabled() throws IOException {+ Directory dir = newDirectory();+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ Document doc = new Document();+ doc.add(new StringField("a", "value", Store.NO));+ doc.add(new StringField("b", "value", Store.NO));+ doc.add(new TextField("c", "value", Store.NO));+ w.addDocument(doc);+ w.commit();++ doc = new Document();+ doc.add(new StringField("a", "value", Store.NO));+ doc.add(new TextField("c", "value", Store.NO));+ w.addDocument(doc);++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);++ Similarity searchSimilarity = randomCompatibleSimilarity();+ searcher.setSimilarity(searchSimilarity);+ TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(10, 10);++ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("value").addField("a", 1.0f).addField("b", 1.0f).build();+ TopDocs topDocs = searcher.search(query, collectorManager);+ assertEquals(new TotalHits(2, TotalHits.Relation.EQUAL_TO), topDocs.totalHits);++ CombinedFieldQuery invalidQuery =+ new CombinedFieldQuery.Builder("value").addField("b", 1.0f).addField("c", 1.0f).build();+ IllegalArgumentException e =+ expectThrows(+ IllegalArgumentException.class, () -> searcher.search(invalidQuery, collectorManager));+ assertTrue(e.getMessage().contains("requires norms to be consistent across fields"));++ reader.close();+ w.close();+ dir.close();+ }++ public void testCopyField() throws IOException {+ Directory dir = newDirectory();+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ int numMatch = atLeast(10);+ int boost1 = Math.max(1, random().nextInt(5));+ int boost2 = Math.max(1, random().nextInt(5));+ for (int i = 0; i < numMatch; i++) {+ Document doc = new Document();+ if (random().nextBoolean()) {+ doc.add(new TextField("a", "baz", Store.NO));+ doc.add(new TextField("b", "baz", Store.NO));+ for (int k = 0; k < boost1 + boost2; k++) {+ doc.add(new TextField("ab", "baz", Store.NO));+ }+ w.addDocument(doc);+ doc.clear();+ }+ int freqA = random().nextInt(5) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo", Store.NO));+ }+ int freqB = random().nextInt(5) + 1;+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "foo", Store.NO));+ }+ int freqAB = freqA * boost1 + freqB * boost2;+ for (int j = 0; j < freqAB; j++) {+ doc.add(new TextField("ab", "foo", Store.NO));+ }+ w.addDocument(doc);+ }+ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);++ searcher.setSimilarity(similarity);+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("foo")+ .addField("a", (float) boost1)+ .addField("b", (float) boost2)+ .build();++ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));++ reader.close();+ w.close();+ dir.close();+ }++ public void testCopyFieldWithSingleField() throws IOException {+ Directory dir = new MMapDirectory(createTempDir());+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ int boost = Math.max(1, random().nextInt(5));+ int numMatch = atLeast(10);+ for (int i = 0; i < numMatch; i++) {+ Document doc = new Document();+ int freqA = random().nextInt(5) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo", Store.NO));+ }++ int freqB = freqA * boost;+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "foo", Store.NO));+ }++ w.addDocument(doc);+ }++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(similarity);+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("foo").addField("a", (float) boost).build();++ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("b", "foo")));++ reader.close();+ w.close();+ dir.close();+ }++ public void testCopyFieldWithMissingFields() throws IOException {+ Directory dir = new MMapDirectory(createTempDir());+ Similarity similarity = randomCompatibleSimilarity();++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ int boost1 = Math.max(1, random().nextInt(5));+ int boost2 = Math.max(1, random().nextInt(5));+ int numMatch = atLeast(10);+ for (int i = 0; i < numMatch; i++) {+ Document doc = new Document();+ int freqA = random().nextInt(5) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo", Store.NO));+ }++ // Choose frequencies such that sometimes we don't add field B+ int freqB = random().nextInt(3);+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "foo", Store.NO));+ }++ int freqAB = freqA * boost1 + freqB * boost2;+ for (int j = 0; j < freqAB; j++) {+ doc.add(new TextField("ab", "foo", Store.NO));+ }++ w.addDocument(doc);+ }++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(similarity);+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("foo")+ .addField("a", (float) boost1)+ .addField("b", (float) boost2)+ .build();++ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));++ reader.close();+ w.close();+ dir.close();+ }++ private static Similarity randomCompatibleSimilarity() {+ return RandomPicks.randomFrom(+ random(),+ Arrays.asList(+ new BM25Similarity(),+ new BooleanSimilarity(),+ new ClassicSimilarity(),+ new LMDirichletSimilarity(),+ new LMJelinekMercerSimilarity(0.1f)));+ }++ private void checkExpectedHits(+ IndexSearcher searcher, int numHits, Query firstQuery, Query secondQuery) throws IOException {+ TopScoreDocCollectorManager collectorManager =+ new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);++ TopDocs firstTopDocs = searcher.search(firstQuery, collectorManager);+ assertEquals(numHits, firstTopDocs.totalHits.value());++ collectorManager = new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);+ TopDocs secondTopDocs = searcher.search(secondQuery, collectorManager);+ CheckHits.checkEqual(firstQuery, secondTopDocs.scoreDocs, firstTopDocs.scoreDocs);+ }++ public void testDocWithNegativeNorms() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(new NegativeNormSimilarity());+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ String queryString = "foo";++ Document doc = new Document();+ // both fields must contain tokens that match the query string "foo"+ doc.add(new TextField("f", "foo", Store.NO));+ doc.add(new TextField("g", "foo baz", Store.NO));+ w.addDocument(doc);++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(new BM25Similarity());+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder(queryString).addField("f").addField("g").build();+ TopDocs topDocs = searcher.search(query, 10);+ CheckHits.checkDocIds("queried docs do not match", new int[] {0}, topDocs.scoreDocs);++ reader.close();+ w.close();+ dir.close();+ }++ public void testMultipleDocsNegativeNorms() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setSimilarity(new NegativeNormSimilarity());+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ String queryString = "foo";++ Document doc0 = new Document();+ doc0.add(new TextField("f", "foo", Store.NO));+ doc0.add(new TextField("g", "foo baz", Store.NO));+ w.addDocument(doc0);++ Document doc1 = new Document();+ // add another match on the query string to the second doc+ doc1.add(new TextField("f", "foo is foo", Store.NO));+ doc1.add(new TextField("g", "foo baz", Store.NO));+ w.addDocument(doc1);++ IndexReader reader = w.getReader();+ IndexSearcher searcher = newSearcher(reader);+ searcher.setSimilarity(new BM25Similarity());+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder(queryString).addField("f").addField("g").build();+ TopDocs topDocs = searcher.search(query, 10);+ // Return doc1 ahead of doc0 since its tf is higher+ CheckHits.checkDocIds("queried docs do not match", new int[] {1, 0}, topDocs.scoreDocs);++ reader.close();+ w.close();+ dir.close();+ }++ private static final class NegativeNormSimilarity extends Similarity {+ @Override+ public long computeNorm(FieldInvertState state) {+ return -128;+ }++ @Override+ public SimScorer scorer(+ float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {+ return new BM25Similarity().scorer(boost, collectionStats, termStats);+ }+ }++ public void testOverrideCollectionStatistics() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig();+ Similarity similarity = randomCompatibleSimilarity();+ iwc.setSimilarity(similarity);+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);++ int numMatch = atLeast(10);+ for (int i = 0; i < numMatch; i++) {+ Document doc = new Document();+ if (random().nextBoolean()) {+ doc.add(new TextField("a", "baz", Store.NO));+ doc.add(new TextField("b", "baz", Store.NO));+ for (int k = 0; k < 2; k++) {+ doc.add(new TextField("ab", "baz", Store.NO));+ }+ w.addDocument(doc);+ doc.clear();+ }+ int freqA = random().nextInt(5) + 1;+ for (int j = 0; j < freqA; j++) {+ doc.add(new TextField("a", "foo", Store.NO));+ }+ int freqB = random().nextInt(5) + 1;+ for (int j = 0; j < freqB; j++) {+ doc.add(new TextField("b", "foo", Store.NO));+ }+ int freqAB = freqA + freqB;+ for (int j = 0; j < freqAB; j++) {+ doc.add(new TextField("ab", "foo", Store.NO));+ }+ w.addDocument(doc);+ }++ IndexReader reader = w.getReader();++ int extraMaxDoc = randomIntBetween(0, 10);+ int extraDocCount = randomIntBetween(0, extraMaxDoc);+ int extraSumDocFreq = extraDocCount + randomIntBetween(0, 10);++ int extraSumTotalTermFreqA = extraSumDocFreq + randomIntBetween(0, 10);+ int extraSumTotalTermFreqB = extraSumDocFreq + randomIntBetween(0, 10);+ int extraSumTotalTermFreqAB = extraSumTotalTermFreqA + extraSumTotalTermFreqB;++ IndexSearcher searcher =+ new IndexSearcher(reader) {+ @Override+ public CollectionStatistics collectionStatistics(String field) throws IOException {+ CollectionStatistics shardStatistics = super.collectionStatistics(field);+ int extraSumTotalTermFreq;+ if (field.equals("a")) {+ extraSumTotalTermFreq = extraSumTotalTermFreqA;+ } else if (field.equals("b")) {+ extraSumTotalTermFreq = extraSumTotalTermFreqB;+ } else if (field.equals("ab")) {+ extraSumTotalTermFreq = extraSumTotalTermFreqAB;+ } else {+ throw new AssertionError("should never be called");+ }+ return new CollectionStatistics(+ field,+ shardStatistics.maxDoc() + extraMaxDoc,+ shardStatistics.docCount() + extraDocCount,+ shardStatistics.sumTotalTermFreq() + extraSumTotalTermFreq,+ shardStatistics.sumDocFreq() + extraSumDocFreq);+ }+ };+ searcher.setSimilarity(similarity);+ CombinedFieldQuery query =+ new CombinedFieldQuery.Builder("foo").addField("a").addField("b").build();++ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));++ reader.close();+ w.close();+ dir.close();+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.javaindex 1ca3f790f43..debd10ea83d 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java@@ -92,7 +92,9 @@ import org.apache.lucene.util.SmallFloat;* org.apache.lucene.search.similarities.BM25Similarity}.** @lucene.experimental+ * @deprecated Use {@link org.apache.lucene.search.CombinedFieldQuery} instead.*/+@Deprecatedpublic final class CombinedFieldQuery extends Query implements Accountable {private static final long BASE_RAM_BYTES =RamUsageEstimator.shallowSizeOfInstance(CombinedFieldQuery.class);@@ -203,7 +205,7 @@ public final class CombinedFieldQuery extends Query implements Accountable {if (pos++ != 0) {builder.append(" ");}- builder.append(term.utf8ToString());+ builder.append(Term.toString(term));}builder.append("))");return builder.toString();