Skip to content

Apache Lucene open-source search software

Replace deprecated search(Query, Collector) with CollectorManager in JoinUtil (#16091)

Luca Cavannae40b4b188e1f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 37d08327c0c..0bedecf21b4 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -390,6 +390,9 @@ Improvements
* GITHUB#16497: Allow Java Panama vector API on GraalVM for JDK 25+. (Abhinav Chinnala)
+* GITHUB#16091: Replace deprecated IndexSearcher#search(Query, Collector) usage with CollectorManager
+ in JoinUtil#createJoinQuery for the GlobalOrdinalsCollector case. (Luca Cavanna)
+
Optimizations
---------------------
* GITHUB#16416: Use ArrayDeque instead of ArrayList for FIFO output buffering in
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
deleted file mode 100644
index 4ae8849f1ea..00000000000
--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.join;
-
-import java.io.IOException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.OrdinalMap;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.Scorable;
-import org.apache.lucene.util.LongBitSet;
-import org.apache.lucene.util.LongValues;
-
-/**
- * A collector that collects all ordinals from a specified field matching the query.
- *
- * @lucene.experimental
- */
-final class GlobalOrdinalsCollector implements Collector {
-
- final String field;
- final LongBitSet collectedOrds;
- final OrdinalMap ordinalMap;
-
- GlobalOrdinalsCollector(String field, OrdinalMap ordinalMap, long valueCount) {
- this.field = field;
- this.ordinalMap = ordinalMap;
- this.collectedOrds = new LongBitSet(valueCount);
- }
-
- public LongBitSet getCollectorOrdinals() {
- return collectedOrds;
- }
-
- @Override
- public org.apache.lucene.search.ScoreMode scoreMode() {
- return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;
- }
-
- @Override
- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
- if (ordinalMap != null) {
- LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
- return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
- } else {
- return new SegmentOrdinalCollector(docTermOrds);
- }
- }
-
- final class OrdinalMapCollector implements LeafCollector {
-
- private final SortedDocValues docTermOrds;
- private final LongValues segmentOrdToGlobalOrdLookup;
-
- OrdinalMapCollector(SortedDocValues docTermOrds, LongValues segmentOrdToGlobalOrdLookup) {
- this.docTermOrds = docTermOrds;
- this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
- }
-
- @Override
- public void collect(int doc) throws IOException {
- if (docTermOrds.advanceExact(doc)) {
- long segmentOrd = docTermOrds.ordValue();
- long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);
- collectedOrds.set(globalOrd);
- }
- }
-
- @Override
- public void setScorer(Scorable scorer) throws IOException {}
- }
-
- final class SegmentOrdinalCollector implements LeafCollector {
-
- private final SortedDocValues docTermOrds;
-
- SegmentOrdinalCollector(SortedDocValues docTermOrds) {
- this.docTermOrds = docTermOrds;
- }
-
- @Override
- public void collect(int doc) throws IOException {
- if (docTermOrds.advanceExact(doc)) {
- collectedOrds.set(docTermOrds.ordValue());
- }
- }
-
- @Override
- public void setScorer(Scorable scorer) throws IOException {}
- }
-}
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.java
new file mode 100644
index 00000000000..3b9b144c60a
--- /dev/null
+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.join;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLongArray;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.OrdinalMap;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.util.LongBitSet;
+import org.apache.lucene.util.LongValues;
+
+/**
+ * A {@link CollectorManager} that collects all ordinals from a specified field matching the query.
+ * All per-slice collectors share a single {@link AtomicLongArray} sized to the global value count,
+ * so memory is O(globalValueCount) regardless of the number of slices or segment cardinality.
+ * Global ordinal remapping happens during collection, and {@link #reduce} assembles the final
+ * {@link LongBitSet} from the shared array.
+ */
+final class GlobalOrdinalsCollectorManager implements CollectorManager<Collector, LongBitSet> {
+
+ private final String field;
+ private final OrdinalMap ordinalMap;
+ private final long valueCount;
+ // Single shared bitset written atomically by all collector slices.
+ private final AtomicLongArray sharedBits;
+
+ GlobalOrdinalsCollectorManager(String field, OrdinalMap ordinalMap, long valueCount) {
+ this.field = field;
+ this.ordinalMap = ordinalMap;
+ this.valueCount = valueCount;
+ this.sharedBits = new AtomicLongArray(LongBitSet.bits2words(valueCount));
+ }
+
+ @Override
+ public Collector newCollector() {
+ return new Collector() {
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+ SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
+ LongValues globalOrds = ordinalMap == null ? null : ordinalMap.getGlobalOrds(context.ord);
+ return new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (docTermOrds.advanceExact(doc)) {
+ long segOrd = docTermOrds.ordValue();
+ setGlobalOrdBit(globalOrds == null ? segOrd : globalOrds.get(segOrd));
+ }
+ }
+ };
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.COMPLETE_NO_SCORES;
+ }
+ };
+ }
+
+ @Override
+ public LongBitSet reduce(Collection<Collector> collectors) {
+ int numWords = sharedBits.length();
+ long[] words = new long[numWords];
+ for (int i = 0; i < numWords; i++) {
+ words[i] = sharedBits.get(i);
+ }
+ return new LongBitSet(words, valueCount);
+ }
+
+ private void setGlobalOrdBit(long globalOrd) {
+ int wordIndex = (int) (globalOrd >> 6);
+ long bit = 1L << globalOrd;
+ long prev = sharedBits.get(wordIndex);
+ while ((prev & bit) == 0) {
+ if (sharedBits.compareAndSet(wordIndex, prev, prev | bit)) {
+ break;
+ }
+ prev = sharedBits.get(wordIndex);
+ }
+ }
+}
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
index df9271fd0fc..756d5916bf0 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
@@ -572,11 +572,10 @@ public final class JoinUtil {
break;
case None:
if (min <= 1 && max == Integer.MAX_VALUE) {
- GlobalOrdinalsCollector globalOrdinalsCollector =
- new GlobalOrdinalsCollector(joinField, ordinalMap, valueCount);
- searcher.search(rewrittenFromQuery, globalOrdinalsCollector);
return new GlobalOrdinalsQuery(
- globalOrdinalsCollector.getCollectorOrdinals(),
+ searcher.search(
+ rewrittenFromQuery,
+ new GlobalOrdinalsCollectorManager(joinField, ordinalMap, valueCount)),
joinField,
ordinalMap,
rewrittenToQuery,
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
index 6e38a3d3323..1d820ef9cbb 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
@@ -91,6 +91,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.search.FixedBitSetCollector;
import org.apache.lucene.tests.search.QueryUtils;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
@@ -283,7 +284,7 @@ public class TestJoinUtil extends LuceneTestCase {
doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
w.addDocument(doc);
- IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
+ IndexSearcher indexSearcher = newSearcher(w.getReader());
w.close();
IndexReader r = indexSearcher.getIndexReader();
@@ -398,7 +399,7 @@ public class TestJoinUtil extends LuceneTestCase {
w.addDocument(doc);
IndexReader r = DirectoryReader.open(w);
- IndexSearcher indexSearcher = new IndexSearcher(r);
+ IndexSearcher indexSearcher = newSearcher(r);
SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
for (int i = 0; i < values.length; i++) {
LeafReader leafReader = r.leaves().get(i).reader();
@@ -493,7 +494,7 @@ public class TestJoinUtil extends LuceneTestCase {
indexSearcher.search(
joinQuery,
new MultiCollectorManager(
- new BitSetCollectorManager(indexSearcher.getIndexReader().maxDoc()),
+ FixedBitSetCollector.createManager(indexSearcher.getIndexReader().maxDoc()),
new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE)));
final BitSet actualResult = (BitSet) searchResults[0];
assertBitSet(expectedResult, actualResult, indexSearcher);
@@ -1583,7 +1584,7 @@ public class TestJoinUtil extends LuceneTestCase {
indexSearcher.search(
joinQuery,
new MultiCollectorManager(
- new BitSetCollectorManager(indexSearcher.getIndexReader().maxDoc()),
+ FixedBitSetCollector.createManager(indexSearcher.getIndexReader().maxDoc()),
new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE)));
// Asserting bit set...
assertBitSet(expectedResult, (BitSet) searchResults[0], indexSearcher);
@@ -2080,31 +2081,6 @@ public class TestJoinUtil extends LuceneTestCase {
}
}
- private static class BitSetCollector extends SimpleCollector {
-
- private final BitSet bitSet;
- private int docBase;
-
- private BitSetCollector(BitSet bitSet) {
- this.bitSet = bitSet;
- }
-
- @Override
- public void collect(int doc) throws IOException {
- bitSet.set(docBase + doc);
- }
-
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- docBase = context.docBase;
- }
-
- @Override
- public org.apache.lucene.search.ScoreMode scoreMode() {
- return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;
- }
- }
-
private abstract static class JoinValueCollector extends SimpleCollector {
final Map<BytesRef, JoinScore> localMap = new HashMap<>();
final String fromField;
@@ -2133,8 +2109,12 @@ public class TestJoinUtil extends LuceneTestCase {
entry.getKey(),
entry.getValue(),
(existing, src) -> {
- if (src.minScore < existing.minScore) existing.minScore = src.minScore;
- if (src.maxScore > existing.maxScore) existing.maxScore = src.maxScore;
+ if (src.minScore < existing.minScore) {
+ existing.minScore = src.minScore;
+ }
+ if (src.maxScore > existing.maxScore) {
+ existing.maxScore = src.maxScore;
+ }
existing.total += src.total;
existing.count += src.count;
return existing;
@@ -2250,28 +2230,4 @@ public class TestJoinUtil extends LuceneTestCase {
return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;
}
}
-
- private static class BitSetCollectorManager
- implements CollectorManager<BitSetCollector, FixedBitSet> {
-
- private final int maxDoc;
-
- BitSetCollectorManager(int maxDoc) {
- this.maxDoc = maxDoc;
- }
-
- @Override
- public BitSetCollector newCollector() {
- return new BitSetCollector(new FixedBitSet(maxDoc));
- }
-
- @Override
- public FixedBitSet reduce(Collection<BitSetCollector> collectors) {
- FixedBitSet result = new FixedBitSet(maxDoc);
- for (BitSetCollector c : collectors) {
- result.or((FixedBitSet) c.bitSet);
- }
- return result;
- }
- }
}