Skip to content

Apache Lucene open-source search software

Rename ToParentBlockJoin[Byte|Float]KnnVectorQuery and adjust to return highest score child doc ID by parent id (#12510)

Benjamin Trent4174b521ddf0
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
similarity index 81%
rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.java
rename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
index 4c13ab8e007..b8c494fa05d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinByteKnnVectorQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
@@ -36,8 +36,20 @@ import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.Bits;
-/** kNN byte vector query that joins matching children vector documents with their parent doc id. */
-public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
+/**
+ * kNN byte vector query that joins matching children vector documents with their parent doc id. The
+ * top documents returned are the child document ids and the calculated scores. Here is how to use
+ * this in conjunction with {@link ToParentBlockJoinQuery}.
+ *
+ * <pre class="prettyprint">
+ * Query knnQuery = new DiversifyingChildrenByteKnnVectorQuery(fieldName, queryVector, ...);
+ * // Rewrite executes kNN search and collects nearest children docIds and their scores
+ * Query rewrittenKnnQuery = searcher.rewrite(knnQuery);
+ * // Join the scored children docs with their parents and score the parents
+ * Query childrenToParents = new ToParentBlockJoinQuery(rewrittenKnnQuery, parentsFilter, ScoreMode.MAX);
+ * </pre>
+ */
+public class DiversifyingChildrenByteKnnVectorQuery extends KnnByteVectorQuery {
private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;
private final BitSetProducer parentsFilter;
@@ -54,7 +66,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
* @param k how many parent documents to return given the matching children
* @param parentsFilter Filter identifying the parent documents.
*/
- public ToParentBlockJoinByteKnnVectorQuery(
+ public DiversifyingChildrenByteKnnVectorQuery(
String field, byte[] query, Query childFilter, int k, BitSetProducer parentsFilter) {
super(field, query, k, childFilter);
this.childFilter = childFilter;
@@ -87,12 +99,11 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
fi.getVectorSimilarityFunction());
HitQueue queue = new HitQueue(k, true);
ScoreDoc topDoc = queue.top();
- int doc;
- while ((doc = vectorScorer.nextParent()) != DocIdSetIterator.NO_MORE_DOCS) {
+ while (vectorScorer.nextParent() != DocIdSetIterator.NO_MORE_DOCS) {
float score = vectorScorer.score();
if (score > topDoc.score) {
topDoc.score = score;
- topDoc.doc = doc;
+ topDoc.doc = vectorScorer.bestChild();
topDoc = queue.updateTop();
}
}
@@ -118,7 +129,8 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
if (parentBitSet == null) {
return NO_RESULTS;
}
- KnnCollector collector = new ToParentJoinKnnCollector(k, visitedLimit, parentBitSet);
+ KnnCollector collector =
+ new DiversifyingNearestChildrenKnnCollector(k, visitedLimit, parentBitSet);
context.reader().searchNearestVectors(field, query, collector, acceptDocs);
return collector.topDocs();
}
@@ -133,7 +145,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
- ToParentBlockJoinByteKnnVectorQuery that = (ToParentBlockJoinByteKnnVectorQuery) o;
+ DiversifyingChildrenByteKnnVectorQuery that = (DiversifyingChildrenByteKnnVectorQuery) o;
return k == that.k
&& Objects.equals(parentsFilter, that.parentsFilter)
&& Objects.equals(childFilter, that.childFilter)
@@ -154,6 +166,7 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
private final DocIdSetIterator acceptedChildrenIterator;
private final BitSet parentBitSet;
private int currentParent = -1;
+ private int bestChild = -1;
private float currentScore = Float.NEGATIVE_INFINITY;
protected ParentBlockJoinByteVectorScorer(
@@ -169,6 +182,10 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
this.parentBitSet = parentBitSet;
}
+ public int bestChild() {
+ return bestChild;
+ }
+
public int nextParent() throws IOException {
int nextChild = acceptedChildrenIterator.docID();
if (nextChild == -1) {
@@ -182,7 +199,11 @@ public class ToParentBlockJoinByteKnnVectorQuery extends KnnByteVectorQuery {
currentParent = parentBitSet.nextSetBit(nextChild);
do {
values.advance(nextChild);
- currentScore = Math.max(currentScore, similarity.compare(query, values.vectorValue()));
+ float score = similarity.compare(query, values.vectorValue());
+ if (score > currentScore) {
+ bestChild = nextChild;
+ currentScore = score;
+ }
} while ((nextChild = acceptedChildrenIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS
&& nextChild < currentParent);
return currentParent;
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
similarity index 80%
rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.java
rename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
index bde0b62733d..e08b3d4c4ff 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinFloatKnnVectorQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
@@ -38,8 +38,18 @@ import org.apache.lucene.util.Bits;
/**
* kNN float vector query that joins matching children vector documents with their parent doc id.
+ * The top documents returned are the child document ids and the calculated scores. Here is how to
+ * use this in conjunction with {@link ToParentBlockJoinQuery}.
+ *
+ * <pre class="prettyprint">
+ * Query knnQuery = new DiversifyingChildrenFloatKnnVectorQuery(fieldName, queryVector, ...);
+ * // Rewrite executes kNN search and collects nearest children docIds and their scores
+ * Query rewrittenKnnQuery = searcher.rewrite(knnQuery);
+ * // Join the scored children docs with their parents and score the parents
+ * Query childrenToParents = new ToParentBlockJoinQuery(rewrittenKnnQuery, parentsFilter, ScoreMode.MAX);
+ * </pre>
*/
-public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
+public class DiversifyingChildrenFloatKnnVectorQuery extends KnnFloatVectorQuery {
private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;
private final BitSetProducer parentsFilter;
@@ -56,7 +66,7 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
* @param k how many parent documents to return given the matching children
* @param parentsFilter Filter identifying the parent documents.
*/
- public ToParentBlockJoinFloatKnnVectorQuery(
+ public DiversifyingChildrenFloatKnnVectorQuery(
String field, float[] query, Query childFilter, int k, BitSetProducer parentsFilter) {
super(field, query, k, childFilter);
this.childFilter = childFilter;
@@ -80,8 +90,8 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
if (parentBitSet == null) {
return NO_RESULTS;
}
- ParentBlockJoinFloatVectorScorer vectorScorer =
- new ParentBlockJoinFloatVectorScorer(
+ DiversifyingChildrenFloatVectorScorer vectorScorer =
+ new DiversifyingChildrenFloatVectorScorer(
context.reader().getFloatVectorValues(field),
acceptIterator,
parentBitSet,
@@ -89,12 +99,11 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
fi.getVectorSimilarityFunction());
HitQueue queue = new HitQueue(k, true);
ScoreDoc topDoc = queue.top();
- int doc;
- while ((doc = vectorScorer.nextParent()) != DocIdSetIterator.NO_MORE_DOCS) {
+ while (vectorScorer.nextParent() != DocIdSetIterator.NO_MORE_DOCS) {
float score = vectorScorer.score();
if (score > topDoc.score) {
topDoc.score = score;
- topDoc.doc = doc;
+ topDoc.doc = vectorScorer.bestChild();
topDoc = queue.updateTop();
}
}
@@ -120,7 +129,8 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
if (parentBitSet == null) {
return NO_RESULTS;
}
- KnnCollector collector = new ToParentJoinKnnCollector(k, visitedLimit, parentBitSet);
+ KnnCollector collector =
+ new DiversifyingNearestChildrenKnnCollector(k, visitedLimit, parentBitSet);
context.reader().searchNearestVectors(field, query, collector, acceptDocs);
return collector.topDocs();
}
@@ -135,7 +145,7 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
- ToParentBlockJoinFloatKnnVectorQuery that = (ToParentBlockJoinFloatKnnVectorQuery) o;
+ DiversifyingChildrenFloatKnnVectorQuery that = (DiversifyingChildrenFloatKnnVectorQuery) o;
return k == that.k
&& Objects.equals(parentsFilter, that.parentsFilter)
&& Objects.equals(childFilter, that.childFilter)
@@ -149,16 +159,17 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
return result;
}
- private static class ParentBlockJoinFloatVectorScorer {
+ private static class DiversifyingChildrenFloatVectorScorer {
private final float[] query;
private final FloatVectorValues values;
private final VectorSimilarityFunction similarity;
private final DocIdSetIterator acceptedChildrenIterator;
private final BitSet parentBitSet;
private int currentParent = -1;
+ private int bestChild = -1;
private float currentScore = Float.NEGATIVE_INFINITY;
- protected ParentBlockJoinFloatVectorScorer(
+ protected DiversifyingChildrenFloatVectorScorer(
FloatVectorValues values,
DocIdSetIterator acceptedChildrenIterator,
BitSet parentBitSet,
@@ -171,6 +182,10 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
this.parentBitSet = parentBitSet;
}
+ public int bestChild() {
+ return bestChild;
+ }
+
public int nextParent() throws IOException {
int nextChild = acceptedChildrenIterator.docID();
if (nextChild == -1) {
@@ -184,7 +199,11 @@ public class ToParentBlockJoinFloatKnnVectorQuery extends KnnFloatVectorQuery {
currentParent = parentBitSet.nextSetBit(nextChild);
do {
values.advance(nextChild);
- currentScore = Math.max(currentScore, similarity.compare(query, values.vectorValue()));
+ float score = similarity.compare(query, values.vectorValue());
+ if (score > currentScore) {
+ bestChild = nextChild;
+ currentScore = score;
+ }
} while ((nextChild = acceptedChildrenIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS
&& nextChild < currentParent);
return currentParent;
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.java
similarity index 69%
rename from lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.java
rename to lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.java
index 669cfabe2db..9a5882d3a2d 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentJoinKnnCollector.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.java
@@ -26,8 +26,11 @@ import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSet;
-/** parent joining knn collector, vector docIds are deduplicated according to the parent bit set. */
-class ToParentJoinKnnCollector extends AbstractKnnCollector {
+/**
+ * This collects the nearest children vectors. Diversifying the results over the provided parent
+ * filter. This means the nearest children vectors are returned, but only one per parent
+ */
+class DiversifyingNearestChildrenKnnCollector extends AbstractKnnCollector {
private final BitSet parentBitSet;
private final NodeIdCachingHeap heap;
@@ -39,7 +42,7 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
* @param visitLimit how many child vectors can be visited
* @param parentBitSet The leaf parent bitset
*/
- public ToParentJoinKnnCollector(int k, int visitLimit, BitSet parentBitSet) {
+ public DiversifyingNearestChildrenKnnCollector(int k, int visitLimit, BitSet parentBitSet) {
super(k, visitLimit);
this.parentBitSet = parentBitSet;
this.heap = new NodeIdCachingHeap(k);
@@ -60,8 +63,8 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
@Override
public boolean collect(int docId, float nodeScore) {
assert !parentBitSet.get(docId);
- int nodeId = parentBitSet.nextSetBit(docId);
- return heap.insertWithOverflow(nodeId, nodeScore);
+ int parentNode = parentBitSet.nextSetBit(docId);
+ return heap.insertWithOverflow(docId, parentNode, nodeScore);
}
@Override
@@ -103,8 +106,7 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
*/
private static class NodeIdCachingHeap {
private final int maxSize;
- private int[] heapNodes;
- private float[] heapScores;
+ private ParentChildScore[] heapNodes;
private int size = 0;
// Used to keep track of nodeId -> positionInHeap. This way when new scores are added for a
@@ -125,36 +127,32 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
this.maxSize = maxSize;
this.nodeIdHeapIndex =
new HashMap<>(maxSize < 2 ? maxSize + 1 : (int) (maxSize / 0.75 + 1.0));
- this.heapNodes = new int[heapSize];
- this.heapScores = new float[heapSize];
+ this.heapNodes = new ParentChildScore[heapSize];
}
public final int topNode() {
- return heapNodes[1];
+ return heapNodes[1].child;
}
public final float topScore() {
- return heapScores[1];
+ return heapNodes[1].score;
}
- private void pushIn(int nodeId, float score) {
+ private void pushIn(int nodeId, int parentId, float score) {
size++;
if (size == heapNodes.length) {
heapNodes = ArrayUtil.grow(heapNodes, (size * 3 + 1) / 2);
- heapScores = ArrayUtil.grow(heapScores, (size * 3 + 1) / 2);
}
- heapNodes[size] = nodeId;
- heapScores[size] = score;
+ heapNodes[size] = new ParentChildScore(nodeId, parentId, score);
upHeap(size);
}
- private void updateElement(int heapIndex, int nodeId, float score) {
- int oldValue = heapNodes[heapIndex];
- assert oldValue == nodeId
+ private void updateElement(int heapIndex, int nodeId, int parentId, float score) {
+ ParentChildScore oldValue = heapNodes[heapIndex];
+ assert oldValue.parent == parentId
: "attempted to update heap element value but with a different node id";
- float oldScore = heapScores[heapIndex];
- heapNodes[heapIndex] = nodeId;
- heapScores[heapIndex] = score;
+ float oldScore = heapNodes[heapIndex].score;
+ heapNodes[heapIndex] = new ParentChildScore(nodeId, parentId, score);
// Since we are a min heap, if the new value is less, we need to make sure to bubble it up
if (score < oldScore) {
upHeap(heapIndex);
@@ -172,26 +170,27 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
*
* @return whether the value was added or updated
*/
- public boolean insertWithOverflow(int node, float score) {
+ public boolean insertWithOverflow(int node, int parentNode, float score) {
if (closed) {
throw new IllegalStateException();
}
- Integer previousNodeIndex = nodeIdHeapIndex.get(node);
+ Integer previousNodeIndex = nodeIdHeapIndex.get(parentNode);
if (previousNodeIndex != null) {
- if (heapScores[previousNodeIndex] < score) {
- updateElement(previousNodeIndex, node, score);
+ if (heapNodes[previousNodeIndex].score < score) {
+ updateElement(previousNodeIndex, node, parentNode, score);
return true;
}
return false;
}
if (size >= maxSize) {
- if (score < heapScores[1] || (score == heapScores[1] && node > heapNodes[1])) {
+ if (score < heapNodes[1].score
+ || (score == heapNodes[1].score && node > heapNodes[1].child)) {
return false;
}
- updateTop(node, score);
+ updateTop(node, parentNode, score);
return true;
}
- pushIn(node, score);
+ pushIn(node, parentNode, score);
return true;
}
@@ -199,7 +198,6 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
closed = true;
if (size > 0) {
heapNodes[1] = heapNodes[size]; // move last to first
- heapScores[1] = heapScores[size]; // move last to first
size--;
downHeapWithoutCacheUpdate(1); // adjust heap
} else {
@@ -207,10 +205,9 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
}
}
- private void updateTop(int nodeId, float score) {
- nodeIdHeapIndex.remove(heapNodes[1]);
- heapNodes[1] = nodeId;
- heapScores[1] = score;
+ private void updateTop(int nodeId, int parentId, float score) {
+ nodeIdHeapIndex.remove(heapNodes[1].parent);
+ heapNodes[1] = new ParentChildScore(nodeId, parentId, score);
downHeap(1);
}
@@ -219,76 +216,82 @@ class ToParentJoinKnnCollector extends AbstractKnnCollector {
return size;
}
- private boolean lessThan(int nodel, float scorel, int noder, float scorer) {
- if (scorel < scorer) {
- return true;
- }
- return scorel == scorer && nodel > noder;
- }
-
private void upHeap(int origPos) {
int i = origPos;
- int bottomNode = heapNodes[i];
- float bottomScore = heapScores[i];
+ ParentChildScore bottomNode = heapNodes[i];
int j = i >>> 1;
- while (j > 0 && lessThan(bottomNode, bottomScore, heapNodes[j], heapScores[j])) {
+ while (j > 0 && bottomNode.compareTo(heapNodes[j]) < 0) {
heapNodes[i] = heapNodes[j];
- heapScores[i] = heapScores[j];
- nodeIdHeapIndex.put(heapNodes[i], i);
+ nodeIdHeapIndex.put(heapNodes[i].parent, i);
i = j;
j = j >>> 1;
}
- nodeIdHeapIndex.put(bottomNode, i);
+ nodeIdHeapIndex.put(bottomNode.parent, i);
heapNodes[i] = bottomNode;
- heapScores[i] = bottomScore;
}
private int downHeap(int i) {
- int node = heapNodes[i];
- float score = heapScores[i];
+ ParentChildScore node = heapNodes[i];
int j = i << 1; // find smaller child
int k = j + 1;
- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {
+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {
j = k;
}
- while (j <= size && lessThan(heapNodes[j], heapScores[j], node, score)) {
+ while (j <= size && heapNodes[j].compareTo(node) < 0) {
heapNodes[i] = heapNodes[j];
- heapScores[i] = heapScores[j];
- nodeIdHeapIndex.put(heapNodes[i], i);
+ nodeIdHeapIndex.put(heapNodes[i].parent, i);
i = j;
j = i << 1;
k = j + 1;
- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {
+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {
j = k;
}
}
- nodeIdHeapIndex.put(node, i);
+ nodeIdHeapIndex.put(node.parent, i);
heapNodes[i] = node; // install saved value
- heapScores[i] = score; // install saved value
return i;
}
private int downHeapWithoutCacheUpdate(int i) {
- int node = heapNodes[i];
- float score = heapScores[i];
+ ParentChildScore node = heapNodes[i];
int j = i << 1; // find smaller child
int k = j + 1;
- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {
+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {
j = k;
}
- while (j <= size && lessThan(heapNodes[j], heapScores[j], node, score)) {
+ while (j <= size && heapNodes[j].compareTo(node) < 0) {
heapNodes[i] = heapNodes[j];
- heapScores[i] = heapScores[j];
i = j;
j = i << 1;
k = j + 1;
- if (k <= size && lessThan(heapNodes[k], heapScores[k], heapNodes[j], heapScores[j])) {
+ if (k <= size && heapNodes[k].compareTo(heapNodes[j]) < 0) {
j = k;
}
}
heapNodes[i] = node; // install saved value
- heapScores[i] = score; // install saved value
return i;
}
}
+
+ /** Keeps track of child node, parent node, and the stored score. */
+ private static class ParentChildScore implements Comparable<ParentChildScore> {
+ private final int parent, child;
+ private final float score;
+
+ ParentChildScore(int child, int parent, float score) {
+ this.child = child;
+ this.parent = parent;
+ this.score = score;
+ }
+
+ @Override
+ public int compareTo(ParentChildScore o) {
+ int fc = Float.compare(score, o.score);
+ if (fc == 0) {
+ // lower numbers are the tiebreakers, lower ids are preferred.
+ return Integer.compare(o.child, child);
+ }
+ return fc;
+ }
+ }
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
index 5868369aa1f..8f8544eac11 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
@@ -201,41 +201,21 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
IndexSearcher searcher = new IndexSearcher(reader);
BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());
Query query = getParentJoinKnnQuery("field", new float[] {2, 2}, null, 3, parentFilter);
- assertScorerResults(
- searcher,
- query,
- new float[] {1f, 1f / 51f},
- new String[] {
- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})
- });
+ assertScorerResults(searcher, query, new float[] {1f, 1f / 51f}, new String[] {"2", "7"});
query = getParentJoinKnnQuery("field", new float[] {6, 6}, null, 3, parentFilter);
assertScorerResults(
- searcher,
- query,
- new float[] {1f / 3f, 1f / 3f},
- new String[] {
- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})
- });
+ searcher, query, new float[] {1f / 3f, 1f / 3f}, new String[] {"5", "7"});
query =
getParentJoinKnnQuery(
"field", new float[] {6, 6}, new MatchAllDocsQuery(), 20, parentFilter);
assertScorerResults(
- searcher,
- query,
- new float[] {1f / 3f, 1f / 3f},
- new String[] {
- encodeInts(new int[] {1, 2, 3, 4, 5}), encodeInts(new int[] {6, 7, 8, 9, 10})
- });
+ searcher, query, new float[] {1f / 3f, 1f / 3f}, new String[] {"5", "7"});
query =
getParentJoinKnnQuery(
"field", new float[] {6, 6}, new MatchAllDocsQuery(), 1, parentFilter);
- assertScorerResults(
- searcher,
- query,
- new float[] {1f / 3f},
- new String[] {encodeInts(new int[] {1, 2, 3, 4, 5})});
+ assertScorerResults(searcher, query, new float[] {1f / 3f}, new String[] {"5"});
}
}
}
@@ -271,8 +251,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
"field", new float[] {0, 0}, null, 8, parentFilter(searcher.getIndexReader())),
10);
assertEquals(8, results.scoreDocs.length);
- assertIdMatches(reader, "[0]", results.scoreDocs[0].doc);
- assertIdMatches(reader, "[7]", results.scoreDocs[7].doc);
+ assertIdMatches(reader, "0", results.scoreDocs[0].doc);
+ assertIdMatches(reader, "7", results.scoreDocs[7].doc);
// test some results in the middle of the sequence - also tests docid tiebreaking
results =
@@ -285,8 +265,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
parentFilter(searcher.getIndexReader())),
10);
assertEquals(8, results.scoreDocs.length);
- assertIdMatches(reader, "[10]", results.scoreDocs[0].doc);
- assertIdMatches(reader, "[6]", results.scoreDocs[7].doc);
+ assertIdMatches(reader, "10", results.scoreDocs[0].doc);
+ assertIdMatches(reader, "6", results.scoreDocs[7].doc);
}
}
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
index 8ebf7a1e2e9..1d1b5c65311 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
@@ -95,9 +95,10 @@ public class TestBlockJoin extends LuceneTestCase {
return job;
}
- private Document makeVector(String vectorField, float[] value) {
+ private Document makeVector(String vectorField, String childsParent, float[] value) {
Document vectorDoc = new Document();
vectorDoc.add(new KnnFloatVectorField(vectorField, value));
+ vectorDoc.add(newStringField("my_parent_id", childsParent, Store.YES));
return vectorDoc;
}
@@ -251,14 +252,14 @@ public class TestBlockJoin extends LuceneTestCase {
final List<Document> docs = new ArrayList<>();
- docs.add(makeVector("vector", new float[] {1f, 2f, 3f}));
- docs.add(makeVector("vector", new float[] {3f, 3f, 3f}));
+ docs.add(makeVector("vector", "parent1", new float[] {1f, 2f, 3f}));
+ docs.add(makeVector("vector", "parent1", new float[] {3f, 3f, 3f}));
docs.add(makeParent("parent1"));
w.addDocuments(docs);
docs.clear();
- docs.add(makeVector("vector", new float[] {0f, 0f, 1f}));
- docs.add(makeVector("vector", new float[] {1f, 1f, 1f}));
+ docs.add(makeVector("vector", "parent2", new float[] {0f, 0f, 1f}));
+ docs.add(makeVector("vector", "parent2", new float[] {1f, 1f, 1f}));
docs.add(makeParent("parent2"));
w.addDocuments(docs);
@@ -271,21 +272,21 @@ public class TestBlockJoin extends LuceneTestCase {
new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));
CheckJoinIndex.check(r, parentsFilter);
- ToParentBlockJoinFloatKnnVectorQuery childKnnJoin =
- new ToParentBlockJoinFloatKnnVectorQuery(
+ DiversifyingChildrenFloatKnnVectorQuery childKnnJoin =
+ new DiversifyingChildrenFloatKnnVectorQuery(
"vector", new float[] {4f, 4f, 4f}, null, 3, parentsFilter);
TopDocs topDocs = s.search(childKnnJoin, 5);
assertEquals(2, topDocs.totalHits.value);
- Document parentDoc = s.storedFields().document(topDocs.scoreDocs[0].doc);
- assertEquals("parent1", parentDoc.get("parent_id"));
+ Document childDoc = s.storedFields().document(topDocs.scoreDocs[0].doc);
+ assertEquals("parent1", childDoc.get("my_parent_id"));
assertEquals(
topDocs.scoreDocs[0].score,
VectorSimilarityFunction.EUCLIDEAN.compare(
new float[] {4f, 4f, 4f}, new float[] {3f, 3f, 3f}),
1e-7);
- parentDoc = s.storedFields().document(topDocs.scoreDocs[1].doc);
- assertEquals("parent2", parentDoc.get("parent_id"));
+ childDoc = s.storedFields().document(topDocs.scoreDocs[1].doc);
+ assertEquals("parent2", childDoc.get("my_parent_id"));
r.close();
dir.close();
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
index 6f5de4383a6..08c3434e8ed 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
@@ -31,7 +31,7 @@ public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVec
Query childFilter,
int k,
BitSetProducer parentBitSet) {
- return new ToParentBlockJoinByteKnnVectorQuery(
+ return new DiversifyingChildrenByteKnnVectorQuery(
fieldName, fromFloat(queryVector), childFilter, k, parentBitSet);
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
index 8281def61fb..775806795ff 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
@@ -43,7 +43,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVe
Query childFilter,
int k,
BitSetProducer parentBitSet) {
- return new ToParentBlockJoinFloatKnnVectorQuery(
+ return new DiversifyingChildrenFloatKnnVectorQuery(
fieldName, queryVector, childFilter, k, parentBitSet);
}
@@ -54,6 +54,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVe
List<Document> toAdd = new ArrayList<>();
Document doc = new Document();
doc.add(getKnnVectorField("field", new float[] {j, j * j}, COSINE));
+ doc.add(newStringField("id", Integer.toString(j), Field.Store.YES));
toAdd.add(doc);
toAdd.add(makeParent(new int[] {j}));
w.addDocuments(toAdd);
@@ -63,8 +64,8 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVe
assertEquals(1, reader.leaves().size());
IndexSearcher searcher = new IndexSearcher(reader);
BitSetProducer parentFilter = parentFilter(searcher.getIndexReader());
- ToParentBlockJoinFloatKnnVectorQuery query =
- new ToParentBlockJoinFloatKnnVectorQuery(
+ DiversifyingChildrenFloatKnnVectorQuery query =
+ new DiversifyingChildrenFloatKnnVectorQuery(
"field", new float[] {2, 3}, null, 3, parentFilter);
/* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then
* normalized by (1 + x) /2.
@@ -78,8 +79,7 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVe
float score1 =
(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);
- assertScorerResults(
- searcher, query, new float[] {score0, score1}, new String[] {"[1]", "[2]"});
+ assertScorerResults(searcher, query, new float[] {score0, score1}, new String[] {"1", "2"});
}
}
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java b/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java
index fdac59fc360..400807bca36 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestToParentJoinKnnResults.java
@@ -18,7 +18,10 @@
package org.apache.lucene.search.join;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -29,7 +32,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {
public void testNeighborsProduct() throws IOException {
// make sure we have the sign correct
BitSet parentBitSet = BitSet.of(new IntArrayDocIdSetIterator(new int[] {1, 3, 5}, 3), 6);
- ToParentJoinKnnCollector nn = new ToParentJoinKnnCollector(2, Integer.MAX_VALUE, parentBitSet);
+ DiversifyingNearestChildrenKnnCollector nn =
+ new DiversifyingNearestChildrenKnnCollector(2, Integer.MAX_VALUE, parentBitSet);
assertTrue(nn.collect(2, 0.5f));
assertTrue(nn.collect(0, 0.2f));
assertTrue(nn.collect(4, 1f));
@@ -43,8 +47,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {
int[] nodes = new int[] {4, 1, 5, 7, 8, 10, 2};
float[] scores = new float[] {1f, 0.5f, 0.6f, 2f, 2f, 1.2f, 4f};
BitSet parentBitSet = BitSet.of(new IntArrayDocIdSetIterator(new int[] {3, 6, 9, 12}, 4), 13);
- ToParentJoinKnnCollector results =
- new ToParentJoinKnnCollector(7, Integer.MAX_VALUE, parentBitSet);
+ DiversifyingNearestChildrenKnnCollector results =
+ new DiversifyingNearestChildrenKnnCollector(7, Integer.MAX_VALUE, parentBitSet);
for (int i = 0; i < nodes.length; i++) {
results.collect(nodes[i], scores[i]);
}
@@ -55,7 +59,7 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {
sortedNodes[i] = topDocs.scoreDocs[i].doc;
sortedScores[i] = topDocs.scoreDocs[i].score;
}
- assertArrayEquals(new int[] {3, 9, 12, 6}, sortedNodes);
+ assertArrayEquals(new int[] {2, 7, 10, 4}, sortedNodes);
assertArrayEquals(new float[] {4f, 2f, 1.2f, 1f}, sortedScores, 0f);
}
@@ -64,8 +68,8 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {
float[] scores = new float[] {1f, 0.5f, 0.6f, 2f, 2f, 3f, 4f, 1f, 0.2f};
BitSet parentBitSet =
BitSet.of(new IntArrayDocIdSetIterator(new int[] {3, 6, 9, 11, 13, 15}, 6), 16);
- ToParentJoinKnnCollector results =
- new ToParentJoinKnnCollector(5, Integer.MAX_VALUE, parentBitSet);
+ DiversifyingNearestChildrenKnnCollector results =
+ new DiversifyingNearestChildrenKnnCollector(5, Integer.MAX_VALUE, parentBitSet);
for (int i = 0; i < nodes.length - 1; i++) {
results.collect(nodes[i], scores[i]);
}
@@ -77,10 +81,35 @@ public class TestToParentJoinKnnResults extends LuceneTestCase {
sortedNodes[i] = topDocs.scoreDocs[i].doc;
sortedScores[i] = topDocs.scoreDocs[i].score;
}
- assertArrayEquals(new int[] {3, 11, 9, 6, 13}, sortedNodes);
+ assertArrayEquals(new int[] {2, 10, 7, 4, 12}, sortedNodes);
assertArrayEquals(new float[] {4f, 3f, 2f, 1f, 1f}, sortedScores, 0f);
}
+ public void testRandomInsertionsWithOverflow() throws IOException {
+ int[] parents = new int[100];
+ List<Integer> children = new ArrayList<>();
+ List<Float> childrenScores = new ArrayList<>();
+ int previousParent = -1;
+ int nextParent = random().nextInt(50) + 2;
+ for (int i = 0; i < 100; i++) {
+ for (int j = previousParent + 1; j < nextParent; j++) {
+ children.add(j);
+ childrenScores.add(random().nextFloat());
+ }
+ parents[i] = nextParent;
+ previousParent = nextParent;
+ nextParent = random().nextInt(50) + 2 + previousParent;
+ }
+ Collections.shuffle(children, random());
+ BitSet parentBitSet =
+ BitSet.of(new IntArrayDocIdSetIterator(parents, parents.length), nextParent + 1);
+ DiversifyingNearestChildrenKnnCollector results =
+ new DiversifyingNearestChildrenKnnCollector(20, Integer.MAX_VALUE, parentBitSet);
+ for (int i = 0; i < children.size(); i++) {
+ results.collect(children.get(i), childrenScores.get(i));
+ }
+ }
+
static class IntArrayDocIdSetIterator extends DocIdSetIterator {
private final int[] docs;