Apache Lucene open-source search software
Rename ToParentBlockJoin[Byte|Float]KnnVectorQuery and adjust to return highest score child doc ID by parent id (#12510)
Benjamin Trent4174b521ddf0
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.javasimilarity index 81%rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.javarename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.javaindex 4c13ab8e007..b8c494fa05d 100644--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.java+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java@@ -36,8 +36,20 @@ import org.apache.lucene.search.TotalHits;import org.apache.lucene.util.BitSet;import org.apache.lucene.util.Bits;-/** kNN byte vector query that joins matching children vector documents with their parent doc id. */-public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {+/**+ * kNN byte vector query that joins matching children vector documents with their parent doc id. The+ * top documents returned are the child document ids and the calculated scores. Here is how to use+ * this in conjunction with {@link ToParentBlockJoinQuery}.+ *+ * <pre class="prettyprint">+ * Query knnQuery = new DiversifyingChildrenByteKnnVectorQuery(fieldName, queryVector, ...);+ * // Rewrite executes kNN search and collects nearest children docIds and their scores+ * Query rewrittenKnnQuery = searcher.rewrite(knnQuery);+ * // Join the scored children docs with their parents and score the parents+ * Query childrenToParents = new ToParentBlockJoinQuery(rewrittenKnnQuery, parentsFilter, ScoreMode.MAX);+ * </pre>+ */+public class DiversifyingChildrenByteKnnVectorQuery extends KnnByteVectorQuery {private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;private final BitSetProducer parentsFilter;@@ -54,7 +66,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {* @param k how many parent documents to return given the matching children* @param parentsFilter Filter identifying the parent documents.*/- public ToParentBlockJoinByteKnnVectorQuery(+ public DiversifyingChildrenByteKnnVectorQuery(String field, byte[] query, Query childFilter, int k, BitSetProducer parentsFilter) {super(field, query, k, childFilter);this.childFilter = childFilter;@@ -87,12 +99,11 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {fi.getVectorSimilarityFunction());HitQueue queue = new HitQueue(k, true);ScoreDoc topDoc = queue.top();- int doc;- while ((doc = vectorScorer.nextParent()) != DocIdSetIterator.NO_MORE_DOCS) {+ while (vectorScorer.nextParent() != DocIdSetIterator.NO_MORE_DOCS) {float score = vectorScorer.score();if (score > topDoc.score) {topDoc.score = score;- topDoc.doc = doc;+ topDoc.doc = vectorScorer.bestChild();topDoc = queue.updateTop();}}@@ -118,7 +129,8 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {if (parentBitSet == null) {return NO_RESULTS;}- KnnCollector collector = new ToParentJoinKnnCollector(k, visitedLimit, parentBitSet);+ KnnCollector collector =+ new DiversifyingNearestChildrenKnnCollector(k, visitedLimit, parentBitSet);context.reader().searchNearestVectors(field, query, collector, acceptDocs);return collector.topDocs();}@@ -133,7 +145,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {if (this == o) return true;if (o == null || getClass() != o.getClass()) return false;if (!super.equals(o)) return false;- ToParentBlockJoinByteKnnVectorQuery that = (ToParentBlockJoinByteKnnVectorQuery) o;+ DiversifyingChildrenByteKnnVectorQuery that = (DiversifyingChildrenByteKnnVectorQuery) o;return k == that.k&& Objects.equals(parentsFilter, that.parentsFilter)&& Objects.equals(childFilter, that.childFilter)@@ -154,6 +166,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {private final DocIdSetIterator acceptedChildrenIterator;private final BitSet parentBitSet;private int currentParent = -1;+ private int bestChild = -1;private float currentScore = Float.NEGATIVE_INFINITY;protected ParentBlockJoinByteVectorScorer(@@ -169,6 +182,10 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {this.parentBitSet = parentBitSet;}+ public int bestChild() {+ return bestChild;+ }+public int nextParent() throws IOException {int nextChild = acceptedChildrenIterator.docID();if (nextChild == -1) {@@ -182,7 +199,11 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {currentParent = parentBitSet.nextSetBit(nextChild);do {values.advance(nextChild);- currentScore = Math.max(currentScore, similarity.compare(query, values.vectorValue()));+ float score = similarity.compare(query, values.vectorValue());+ if (score > currentScore) {+ bestChild = nextChild;+ currentScore = score;+ }} while ((nextChild = acceptedChildrenIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS&& nextChild < currentParent);return currentParent;diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.javasimilarity index 80%rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.javarename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.javaindex bde0b62733d..e08b3d4c4ff 100644--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.java+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java@@ -38,8 +38,18 @@ import org.apache.lucene.util.Bits;/*** kNN float vector query that joins matching children vector documents with their parent doc id.+ * The top documents returned are the child document ids and the calculated scores. Here is how to+ * use this in conjunction with {@link ToParentBlockJoinQuery}.+ *+ * <pre class="prettyprint">+ * Query knnQuery = new DiversifyingChildrenFloatKnnVectorQuery(fieldName, queryVector, ...);+ * // Rewrite executes kNN search and collects nearest children docIds and their scores+ * Query rewrittenKnnQuery = searcher.rewrite(knnQuery);+ * // Join the scored children docs with their parents and score the parents+ * Query childrenToParents = new ToParentBlockJoinQuery(rewrittenKnnQuery, parentsFilter, ScoreMode.MAX);+ * </pre>*/-public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {+public class DiversifyingChildrenFloatKnnVectorQuery extends KnnFloatVectorQuery {private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;private final BitSetProducer parentsFilter;@@ -56,7 +66,7 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {* @param k how many parent documents to return given the matching children* @param parentsFilter Filter identifying the parent documents.*/- public ToParentBlockJoinFloatKnnVectorQuery(+ public DiversifyingChildrenFloatKnnVectorQuery(String field, float[] query, Query childFilter, int k, BitSetProducer parentsFilter) {super(field, query, k, childFilter);this.childFilter = childFilter;@@ -80,8 +90,8 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {if (parentBitSet == null) {return NO_RESULTS;}- ParentBlockJoinFloatVectorScorer vectorScorer =- new ParentBlockJoinFloatVectorScorer(+ DiversifyingChildrenFloatVectorScorer vectorScorer =+ new DiversifyingChildrenFloatVectorScorer(context.reader().getFloatVectorValues(field),acceptIterator,parentBitSet,@@ -89,12 +99,11 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {fi.getVectorSimilarityFunction());HitQueue queue = new HitQueue(k, true);ScoreDoc topDoc = queue.top();- int doc;- while ((doc = vectorScorer.nextParent()) != DocIdSetIterator.NO_MORE_DOCS) {+ while (vectorScorer.nextParent() != DocIdSetIterator.NO_MORE_DOCS) {float score = vectorScorer.score();if (score > topDoc.score) {topDoc.score = score;- topDoc.doc = doc;+ topDoc.doc = vectorScorer.bestChild();topDoc = queue.updateTop();}}@@ -120,7 +129,8 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {if (parentBitSet == null) {return NO_RESULTS;}- KnnCollector collector = new ToParentJoinKnnCollector(k, visitedLimit, parentBitSet);+ KnnCollector collector =+ new DiversifyingNearestChildrenKnnCollector(k, visitedLimit, parentBitSet);context.reader().searchNearestVectors(field, query, collector, acceptDocs);return collector.topDocs();}@@ -135,7 +145,7 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {if (this == o) return true;if (o == null || getClass() != o.getClass()) return false;if (!super.equals(o)) return false;- ToParentBlockJoinFloatKnnVectorQuery that = (ToParentBlockJoinFloatKnnVectorQuery) o;+ DiversifyingChildrenFloatKnnVectorQuery that = (DiversifyingChildrenFloatKnnVectorQuery) o;return k == that.k&& Objects.equals(parentsFilter, that.parentsFilter)&& Objects.equals(childFilter, that.childFilter)@@ -149,16 +159,17 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {return result;}- private static class ParentBlockJoinFloatVectorScorer {+ private static class DiversifyingChildrenFloatVectorScorer {private final float[] query;private final FloatVectorValues values;private final VectorSimilarityFunction similarity;private final DocIdSetIterator acceptedChildrenIterator;private final BitSet parentBitSet;private int currentParent = -1;+ private int bestChild = -1;private float currentScore = Float.NEGATIVE_INFINITY;- protected ParentBlockJoinFloatVectorScorer(+ protected DiversifyingChildrenFloatVectorScorer(FloatVectorValues values,DocIdSetIterator acceptedChildrenIterator,BitSet parentBitSet,@@ -171,6 +182,10 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {this.parentBitSet = parentBitSet;}+ public int bestChild() {+ return bestChild;+ }+public int nextParent() throws IOException {int nextChild = acceptedChildrenIterator.docID();if (nextChild == -1) {@@ -184,7 +199,11 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {currentParent = parentBitSet.nextSetBit(nextChild);do {values.advance(nextChild);- currentScore = Math.max(currentScore, similarity.compare(query, values.vectorValue()));+ float score = similarity.compare(query, values.vectorValue());+ if (score > currentScore) {+ bestChild = nextChild;+ currentScore = score;+ }} while ((nextChild = acceptedChildrenIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS&& nextChild < currentParent);return currentParent;diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.javasimilarity index 69%rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.javarename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.javaindex 669cfabe2db..9a5882d3a2d 100644--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.java+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.java@@ -26,8 +26,11 @@ import org.apache.lucene.search.TotalHits;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitSet;-/** parent joining knn collector, vector docIds are deduplicated according to the parent bit set. */-class ToParentJoinKnnCollector extends AbstractKnnCollector {+/**+ * This collects the nearest children vectors. Diversifying the results over the provided parent+ * filter. This means the nearest children vectors are returned, but only one per parent+ */+class DiversifyingNearestChildrenKnnCollector extends AbstractKnnCollector {private final BitSet parentBitSet;private final NodeIdCachingHeap heap;@@ -39,7 +42,7 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {* @param visitLimit how many child vectors can be visited* @param parentBitSet The leaf parent bitset*/- public ToParentJoinKnnCollector(int k, int visitLimit, BitSet parentBitSet) {+ public DiversifyingNearestChildrenKnnCollector(int k, int visitLimit, BitSet parentBitSet) {super(k, visitLimit);this.parentBitSet = parentBitSet;this.heap = new NodeIdCachingHeap(k);@@ -60,8 +63,8 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {@Overridepublic boolean collect(int docId, float nodeScore) {assert !parentBitSet.get(docId);- int nodeId = parentBitSet.nextSetBit(docId);- return heap.insertWithOverflow(nodeId, nodeScore);+ int parentNode = parentBitSet.nextSetBit(docId);+ return heap.insertWithOverflow(docId, parentNode, nodeScore);}@Override@@ -103,8 +106,7 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {*/private static class NodeIdCachingHeap {private final int maxSize;- private int[] heapNodes;- private float[] heapScores;+ private ParentChildScore[] heapNodes;private int size = 0;// Used to keep track of nodeId -> positionInHeap. This way when new scores are added for a@@ -125,36 +127,32 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {this.maxSize = maxSize;this.nodeIdHeapIndex =new HashMap<>(maxSize < 2 ? maxSize + 1 : (int) (maxSize / 0.75 + 1.0));- this.heapNodes = new int[heapSize];- this.heapScores = new float[heapSize];+ this.heapNodes = new ParentChildScore[heapSize];}public final int topNode() {- return heapNodes[1];+ return heapNodes[1].child;}public final float topScore() {- return heapScores[1];+ return heapNodes[1].score;}- private void pushIn(int nodeId, float score) {+ private void pushIn(int nodeId, int parentId, float score) {size++;if (size == heapNodes.length) {heapNodes = ArrayUtil.grow(heapNodes, (size * 3 + 1) / 2);- heapScores = ArrayUtil.grow(heapScores, (size * 3 + 1) / 2);}- heapNodes[size] = nodeId;- heapScores[size] = score;+ heapNodes[size] = new ParentChildScore(nodeId, parentId, score);upHeap(size);}- private void updateElement(int heapIndex, int nodeId, float score) {- int oldValue = heapNodes[heapIndex];- assert oldValue == nodeId+ private void updateElement(int heapIndex, int nodeId, int parentId, float score) {+ ParentChildScore oldValue = heapNodes[heapIndex];+ assert oldValue.parent == parentId: "attempted to update heap element value but with a different node id";- float oldScore = heapScores[heapIndex];- heapNodes[heapIndex] = nodeId;- heapScores[heapIndex] = score;+ float oldScore = heapNodes[heapIndex].score;+ heapNodes[heapIndex] = new ParentChildScore(nodeId, parentId, score);// Since we are a min heap, if the new value is less, we need to make sure to bubble it upif (score < oldScore) {upHeap(heapIndex);@@ -172,26 +170,27 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {** @return whether the value was added or updated*/- public boolean insertWithOverflow(int node, float score) {+ public boolean insertWithOverflow(int node, int parentNode, float score) {if (closed) {throw new IllegalStateException();}- Integer previousNodeIndex = nodeIdHeapIndex.get(node);+ Integer previousNodeIndex = nodeIdHeapIndex.get(parentNode);if (previousNodeIndex != null) {- if (heapScores[previousNodeIndex] < score) {- updateElement(previousNodeIndex, node, score);+ if (heapNodes[previousNodeIndex].score < score) {+ updateElement(previousNodeIndex, node, parentNode, score);return true;}return false;}if (size >= maxSize) {- if (score < heapScores[1] || (score == heapScores[1] && node > heapNodes[1])) {+ if (score < heapNodes[1].score+ || (score == heapNodes[1].score && node > heapNodes[1].child)) {return false;}- updateTop(node, score);+ updateTop(node, parentNode, score);return true;}- pushIn(node, score);+ pushIn(node, parentNode, score);return true;}@@ -199,7 +198,6 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {closed = true;if (size > 0) {heapNodes[1] = heapNodes[size]; // move last to first- heapScores[1] = heapScores[size]; // move last to firstsize--;downHeapWithoutCacheUpdate(1); // adjust heap} else {@@ -207,10 +205,9 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {}}- private void updateTop(int nodeId, float score) {- nodeIdHeapIndex.remove(heapNodes[1]);- heapNodes[1] = nodeId;- heapScores[1] = score;+ private void updateTop(int nodeId, int parentId, float score) {+ nodeIdHeapIndex.remove(heapNodes[1].parent);+ heapNodes[1] = new ParentChildScore(nodeId, parentId, score);downHeap(1);}@@ -219,76 +216,82 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {return size;}- private boolean lessThan(int nodel, float scorel, int noder, float scorer) {- if (scorel < scorer) {- return true;- }- return scorel == scorer && nodel > noder;- }-private void upHeap(int origPos) {int i = origPos;- int bottomNode = heapNodes[i];- float bottomScore = heapScores[i];+ ParentChildScore bottomNode = heapNodes[i];int j = i >>> 1;- while (j > 0 && lessThan(bottomNode, bottomScore, heapNodes[j], heapScores[j])) {+ while (j > 0 && bottomNode.compareTo(heapNodes[j]) < 0) {heapNodes[i] = heapNodes[j];- heapScores[i] = heapScores[j];- nodeIdHeapIndex.put(heapNodes[i], i);+ nodeIdHeapIndex.put(heapNodes[i].parent, i);i = j;j = j >>> 1;}- nodeIdHeapIndex.put(bottomNode, i);+ nodeIdHeapIndex.put(bottomNode.parent, i);heapNodes[i] = bottomNode;- heapScores[i] = bottomScore;}private int downHeap(int i) {- int node = heapNodes[i];- float score = heapScores[i];+ ParentChildScore node = heapNodes[i];int j = i << 1; // find smaller childint k = j + 1;- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {j = k;}- while (j <= size && lessThan(heapNodes[j], heapScores[j], node, score)) {+ while (j <= size && heapNodes[j].compareTo(node) < 0) {heapNodes[i] = heapNodes[j];- heapScores[i] = heapScores[j];- nodeIdHeapIndex.put(heapNodes[i], i);+ nodeIdHeapIndex.put(heapNodes[i].parent, i);i = j;j = i << 1;k = j + 1;- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {j = k;}}- nodeIdHeapIndex.put(node, i);+ nodeIdHeapIndex.put(node.parent, i);heapNodes[i] = node; // install saved value- heapScores[i] = score; // install saved valuereturn i;}private int downHeapWithoutCacheUpdate(int i) {- int node = heapNodes[i];- float score = heapScores[i];+ ParentChildScore node = heapNodes[i];int j = i << 1; // find smaller childint k = j + 1;- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {j = k;}- while (j <= size && lessThan(heapNodes[j], heapScores[j], node, score)) {+ while (j <= size && heapNodes[j].compareTo(node) < 0) {heapNodes[i] = heapNodes[j];- heapScores[i] = heapScores[j];i = j;j = i << 1;k = j + 1;- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {j = k;}}heapNodes[i] = node; // install saved value- heapScores[i] = score; // install saved valuereturn i;}}++ /** Keeps track of child node, parent node, and the stored score. */+ private static class ParentChildScore implements Comparable<ParentChildScore> {+ private final int parent, child;+ private final float score;++ ParentChildScore(int child, int parent, float score) {+ this.child = child;+ this.parent = parent;+ this.score = score;+ }++ @Override+ public int compareTo(ParentChildScore o) {+ int fc = Float.compare(score, o.score);+ if (fc == 0) {+ // lower numbers are the tiebreakers, lower ids are preferred.+ return Integer.compare(o.child, child);+ }+ return fc;+ }+ }}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.javaindex 5868369aa1f..8f8544eac11 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java@@ -201,41 +201,21 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {IndexSearcher searcher = new IndexSearcher(reader);BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());Query query = getParentJoinKnnQuery("field", new float[] {2, 2}, null, 3, parentFilter);- assertScorerResults(- searcher,- query,- new float[] {1f, 1f / 51f},- new String[] {- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})- });+ assertScorerResults(searcher, query, new float[] {1f, 1f / 51f}, new String[] {"2", "7"});query = getParentJoinKnnQuery("field", new float[] {6, 6}, null, 3, parentFilter);assertScorerResults(- searcher,- query,- new float[] {1f / 3f, 1f / 3f},- new String[] {- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})- });+ searcher, query, new float[] {1f / 3f, 1f / 3f}, new String[] {"5", "7"});query =getParentJoinKnnQuery("field", new float[] {6, 6}, new MatchAllDocsQuery(), 20, parentFilter);assertScorerResults(- searcher,- query,- new float[] {1f / 3f, 1f / 3f},- new String[] {- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})- });+ searcher, query, new float[] {1f / 3f, 1f / 3f}, new String[] {"5", "7"});query =getParentJoinKnnQuery("field", new float[] {6, 6}, new MatchAllDocsQuery(), 1, parentFilter);- assertScorerResults(- searcher,- query,- new float[] {1f / 3f},- new String[] {encodeInts(new int[] {1, 2, 3, 4, 5})});+ assertScorerResults(searcher, query, new float[] {1f / 3f}, new String[] {"5"});}}}@@ -271,8 +251,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {"field", new float[] {0, 0}, null, 8, parentFilter(searcher.getIndexReader())),10);assertEquals(8, results.scoreDocs.length);- assertIdMatches(reader, "[0]", results.scoreDocs[0].doc);- assertIdMatches(reader, "[7]", results.scoreDocs[7].doc);+ assertIdMatches(reader, "0", results.scoreDocs[0].doc);+ assertIdMatches(reader, "7", results.scoreDocs[7].doc);// test some results in the middle of the sequence - also tests docid tiebreakingresults =@@ -285,8 +265,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {parentFilter(searcher.getIndexReader())),10);assertEquals(8, results.scoreDocs.length);- assertIdMatches(reader, "[10]", results.scoreDocs[0].doc);- assertIdMatches(reader, "[6]", results.scoreDocs[7].doc);+ assertIdMatches(reader, "10", results.scoreDocs[0].doc);+ assertIdMatches(reader, "6", results.scoreDocs[7].doc);}}}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.javaindex 8ebf7a1e2e9..1d1b5c65311 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java@@ -95,9 +95,10 @@ public class TestBlockJoin extends LuceneTestCase {return job;}- private Document makeVector(String vectorField, float[] value) {+ private Document makeVector(String vectorField, String childsParent, float[] value) {Document vectorDoc = new Document();vectorDoc.add(new KnnFloatVectorField(vectorField, value));+ vectorDoc.add(newStringField("my_parent_id", childsParent, Store.YES));return vectorDoc;}@@ -251,14 +252,14 @@ public class TestBlockJoin extends LuceneTestCase {final List<Document> docs = new ArrayList<>();- docs.add(makeVector("vector", new float[] {1f, 2f, 3f}));- docs.add(makeVector("vector", new float[] {3f, 3f, 3f}));+ docs.add(makeVector("vector", "parent1", new float[] {1f, 2f, 3f}));+ docs.add(makeVector("vector", "parent1", new float[] {3f, 3f, 3f}));docs.add(makeParent("parent1"));w.addDocuments(docs);docs.clear();- docs.add(makeVector("vector", new float[] {0f, 0f, 1f}));- docs.add(makeVector("vector", new float[] {1f, 1f, 1f}));+ docs.add(makeVector("vector", "parent2", new float[] {0f, 0f, 1f}));+ docs.add(makeVector("vector", "parent2", new float[] {1f, 1f, 1f}));docs.add(makeParent("parent2"));w.addDocuments(docs);@@ -271,21 +272,21 @@ public class TestBlockJoin extends LuceneTestCase {new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));CheckJoinIndex.check(r, parentsFilter);- ToParentBlockJoinFloatKnnVectorQuery childKnnJoin =- new ToParentBlockJoinFloatKnnVectorQuery(+ DiversifyingChildrenFloatKnnVectorQuery childKnnJoin =+ new DiversifyingChildrenFloatKnnVectorQuery("vector", new float[] {4f, 4f, 4f}, null, 3, parentsFilter);TopDocs topDocs = s.search(childKnnJoin, 5);assertEquals(2, topDocs.totalHits.value);- Document parentDoc = s.storedFields().document(topDocs.scoreDocs[0].doc);- assertEquals("parent1", parentDoc.get("parent_id"));+ Document childDoc = s.storedFields().document(topDocs.scoreDocs[0].doc);+ assertEquals("parent1", childDoc.get("my_parent_id"));assertEquals(topDocs.scoreDocs[0].score,VectorSimilarityFunction.EUCLIDEAN.compare(new float[] {4f, 4f, 4f}, new float[] {3f, 3f, 3f}),1e-7);- parentDoc = s.storedFields().document(topDocs.scoreDocs[1].doc);- assertEquals("parent2", parentDoc.get("parent_id"));+ childDoc = s.storedFields().document(topDocs.scoreDocs[1].doc);+ assertEquals("parent2", childDoc.get("my_parent_id"));r.close();dir.close();}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.javaindex 6f5de4383a6..08c3434e8ed 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java@@ -31,7 +31,7 @@ public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVecQuery childFilter,int k,BitSetProducer parentBitSet) {- return new ToParentBlockJoinByteKnnVectorQuery(+ return new DiversifyingChildrenByteKnnVectorQuery(fieldName, fromFloat(queryVector), childFilter, k, parentBitSet);}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.javaindex 8281def61fb..775806795ff 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java@@ -43,7 +43,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVeQuery childFilter,int k,BitSetProducer parentBitSet) {- return new ToParentBlockJoinFloatKnnVectorQuery(+ return new DiversifyingChildrenFloatKnnVectorQuery(fieldName, queryVector, childFilter, k, parentBitSet);}@@ -54,6 +54,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVeList<Document> toAdd = new ArrayList<>();Document doc = new Document();doc.add(getKnnVectorField("field", new float[] {j, j * j}, COSINE));+ doc.add(newStringField("id", Integer.toString(j), Field.Store.YES));toAdd.add(doc);toAdd.add(makeParent(new int[] {j}));w.addDocuments(toAdd);@@ -63,8 +64,8 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVeassertEquals(1, reader.leaves().size());IndexSearcher searcher = new IndexSearcher(reader);BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());- ToParentBlockJoinFloatKnnVectorQuery query =- new ToParentBlockJoinFloatKnnVectorQuery(+ DiversifyingChildrenFloatKnnVectorQuery query =+ new DiversifyingChildrenFloatKnnVectorQuery("field", new float[] {2, 3}, null, 3, parentFilter);/* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then* normalized by (1 + x) /2.@@ -78,8 +79,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVefloat score1 =(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);- assertScorerResults(- searcher, query, new float[] {score0, score1}, new String[] {"[1]", "[2]"});+ assertScorerResults(searcher, query, new float[] {score0, score1}, new String[] {"1", "2"});}}}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java b/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.javaindex fdac59fc360..400807bca36 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java@@ -18,7 +18,10 @@package org.apache.lucene.search.join;import java.io.IOException;+import java.util.ArrayList;import java.util.Arrays;+import java.util.Collections;+import java.util.List;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.TopDocs;import org.apache.lucene.tests.util.LuceneTestCase;@@ -29,7 +32,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {public void testNeighborsProduct() throws IOException {// make sure we have the sign correctBitSet parentBitSet = BitSet.of(new IntArrayDocIdSetIterator(new int[] {1, 3, 5}, 3), 6);- ToParentJoinKnnCollector nn = new ToParentJoinKnnCollector(2, Integer.MAX_VALUE, parentBitSet);+ DiversifyingNearestChildrenKnnCollector nn =+ new DiversifyingNearestChildrenKnnCollector(2, Integer.MAX_VALUE, parentBitSet);assertTrue(nn.collect(2, 0.5f));assertTrue(nn.collect(0, 0.2f));assertTrue(nn.collect(4, 1f));@@ -43,8 +47,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {int[] nodes = new int[] {4, 1, 5, 7, 8, 10, 2};float[] scores = new float[] {1f, 0.5f, 0.6f, 2f, 2f, 1.2f, 4f};BitSet parentBitSet = BitSet.of(new IntArrayDocIdSetIterator(new int[] {3, 6, 9, 12}, 4), 13);- ToParentJoinKnnCollector results =- new ToParentJoinKnnCollector(7, Integer.MAX_VALUE, parentBitSet);+ DiversifyingNearestChildrenKnnCollector results =+ new DiversifyingNearestChildrenKnnCollector(7, Integer.MAX_VALUE, parentBitSet);for (int i = 0; i < nodes.length; i++) {results.collect(nodes[i], scores[i]);}@@ -55,7 +59,7 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {sortedNodes[i] = topDocs.scoreDocs[i].doc;sortedScores[i] = topDocs.scoreDocs[i].score;}- assertArrayEquals(new int[] {3, 9, 12, 6}, sortedNodes);+ assertArrayEquals(new int[] {2, 7, 10, 4}, sortedNodes);assertArrayEquals(new float[] {4f, 2f, 1.2f, 1f}, sortedScores, 0f);}@@ -64,8 +68,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {float[] scores = new float[] {1f, 0.5f, 0.6f, 2f, 2f, 3f, 4f, 1f, 0.2f};BitSet parentBitSet =BitSet.of(new IntArrayDocIdSetIterator(new int[] {3, 6, 9, 11, 13, 15}, 6), 16);- ToParentJoinKnnCollector results =- new ToParentJoinKnnCollector(5, Integer.MAX_VALUE, parentBitSet);+ DiversifyingNearestChildrenKnnCollector results =+ new DiversifyingNearestChildrenKnnCollector(5, Integer.MAX_VALUE, parentBitSet);for (int i = 0; i < nodes.length - 1; i++) {results.collect(nodes[i], scores[i]);}@@ -77,10 +81,35 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {sortedNodes[i] = topDocs.scoreDocs[i].doc;sortedScores[i] = topDocs.scoreDocs[i].score;}- assertArrayEquals(new int[] {3, 11, 9, 6, 13}, sortedNodes);+ assertArrayEquals(new int[] {2, 10, 7, 4, 12}, sortedNodes);assertArrayEquals(new float[] {4f, 3f, 2f, 1f, 1f}, sortedScores, 0f);}+ public void testRandomInsertionsWithOverflow() throws IOException {+ int[] parents = new int[100];+ List<Integer> children = new ArrayList<>();+ List<Float> childrenScores = new ArrayList<>();+ int previousParent = -1;+ int nextParent = random().nextInt(50) + 2;+ for (int i = 0; i < 100; i++) {+ for (int j = previousParent + 1; j < nextParent; j++) {+ children.add(j);+ childrenScores.add(random().nextFloat());+ }+ parents[i] = nextParent;+ previousParent = nextParent;+ nextParent = random().nextInt(50) + 2 + previousParent;+ }+ Collections.shuffle(children, random());+ BitSet parentBitSet =+ BitSet.of(new IntArrayDocIdSetIterator(parents, parents.length), nextParent + 1);+ DiversifyingNearestChildrenKnnCollector results =+ new DiversifyingNearestChildrenKnnCollector(20, Integer.MAX_VALUE, parentBitSet);+ for (int i = 0; i < children.size(); i++) {+ results.collect(children.get(i), childrenScores.get(i));+ }+ }+static class IntArrayDocIdSetIterator extends DocIdSetIterator {private final int[] docs;