Apache Lucene open-source search software
Replace deprecated search(Query, Collector) with CollectorManager in JoinUtil (#16091)
Luca Cavannae40b4b188e1f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 37d08327c0c..0bedecf21b4 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -390,6 +390,9 @@ Improvements* GITHUB#16497: Allow Java Panama vector API on GraalVM for JDK 25+. (Abhinav Chinnala)+* GITHUB#16091: Replace deprecated IndexSearcher#search(Query, Collector) usage with CollectorManager+ in JoinUtil#createJoinQuery for the GlobalOrdinalsCollector case. (Luca Cavanna)+Optimizations---------------------* GITHUB#16416: Use ArrayDeque instead of ArrayList for FIFO output buffering indiff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.javadeleted file mode 100644index 4ae8849f1ea..00000000000--- a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java+++ /dev/null@@ -1,108 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.search.join;--import java.io.IOException;-import org.apache.lucene.index.DocValues;-import org.apache.lucene.index.LeafReaderContext;-import org.apache.lucene.index.OrdinalMap;-import org.apache.lucene.index.SortedDocValues;-import org.apache.lucene.search.Collector;-import org.apache.lucene.search.LeafCollector;-import org.apache.lucene.search.Scorable;-import org.apache.lucene.util.LongBitSet;-import org.apache.lucene.util.LongValues;--/**- * A collector that collects all ordinals from a specified field matching the query.- *- * @lucene.experimental- */-final class GlobalOrdinalsCollector implements Collector {-- final String field;- final LongBitSet collectedOrds;- final OrdinalMap ordinalMap;-- GlobalOrdinalsCollector(String field, OrdinalMap ordinalMap, long valueCount) {- this.field = field;- this.ordinalMap = ordinalMap;- this.collectedOrds = new LongBitSet(valueCount);- }-- public LongBitSet getCollectorOrdinals() {- return collectedOrds;- }-- @Override- public org.apache.lucene.search.ScoreMode scoreMode() {- return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;- }-- @Override- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {- SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);- if (ordinalMap != null) {- LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);- return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);- } else {- return new SegmentOrdinalCollector(docTermOrds);- }- }-- final class OrdinalMapCollector implements LeafCollector {-- private final SortedDocValues docTermOrds;- private final LongValues segmentOrdToGlobalOrdLookup;-- OrdinalMapCollector(SortedDocValues docTermOrds, LongValues segmentOrdToGlobalOrdLookup) {- this.docTermOrds = docTermOrds;- this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;- }-- @Override- public void collect(int doc) throws IOException {- if (docTermOrds.advanceExact(doc)) {- long segmentOrd = docTermOrds.ordValue();- long globalOrd = segmentOrdToGlobalOrdLookup.get(segmentOrd);- collectedOrds.set(globalOrd);- }- }-- @Override- public void setScorer(Scorable scorer) throws IOException {}- }-- final class SegmentOrdinalCollector implements LeafCollector {-- private final SortedDocValues docTermOrds;-- SegmentOrdinalCollector(SortedDocValues docTermOrds) {- this.docTermOrds = docTermOrds;- }-- @Override- public void collect(int doc) throws IOException {- if (docTermOrds.advanceExact(doc)) {- collectedOrds.set(docTermOrds.ordValue());- }- }-- @Override- public void setScorer(Scorable scorer) throws IOException {}- }-}diff --git a/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.java b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.javanew file mode 100644index 00000000000..3b9b144c60a--- /dev/null+++ b/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollectorManager.java@@ -0,0 +1,105 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search.join;++import java.io.IOException;+import java.util.Collection;+import java.util.concurrent.atomic.AtomicLongArray;+import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.OrdinalMap;+import org.apache.lucene.index.SortedDocValues;+import org.apache.lucene.search.Collector;+import org.apache.lucene.search.CollectorManager;+import org.apache.lucene.search.LeafCollector;+import org.apache.lucene.search.Scorable;+import org.apache.lucene.search.ScoreMode;+import org.apache.lucene.util.LongBitSet;+import org.apache.lucene.util.LongValues;++/**+ * A {@link CollectorManager} that collects all ordinals from a specified field matching the query.+ * All per-slice collectors share a single {@link AtomicLongArray} sized to the global value count,+ * so memory is O(globalValueCount) regardless of the number of slices or segment cardinality.+ * Global ordinal remapping happens during collection, and {@link #reduce} assembles the final+ * {@link LongBitSet} from the shared array.+ */+final class GlobalOrdinalsCollectorManager implements CollectorManager<Collector, LongBitSet> {++ private final String field;+ private final OrdinalMap ordinalMap;+ private final long valueCount;+ // Single shared bitset written atomically by all collector slices.+ private final AtomicLongArray sharedBits;++ GlobalOrdinalsCollectorManager(String field, OrdinalMap ordinalMap, long valueCount) {+ this.field = field;+ this.ordinalMap = ordinalMap;+ this.valueCount = valueCount;+ this.sharedBits = new AtomicLongArray(LongBitSet.bits2words(valueCount));+ }++ @Override+ public Collector newCollector() {+ return new Collector() {+ @Override+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {+ SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);+ LongValues globalOrds = ordinalMap == null ? null : ordinalMap.getGlobalOrds(context.ord);+ return new LeafCollector() {+ @Override+ public void setScorer(Scorable scorer) {}++ @Override+ public void collect(int doc) throws IOException {+ if (docTermOrds.advanceExact(doc)) {+ long segOrd = docTermOrds.ordValue();+ setGlobalOrdBit(globalOrds == null ? segOrd : globalOrds.get(segOrd));+ }+ }+ };+ }++ @Override+ public ScoreMode scoreMode() {+ return ScoreMode.COMPLETE_NO_SCORES;+ }+ };+ }++ @Override+ public LongBitSet reduce(Collection<Collector> collectors) {+ int numWords = sharedBits.length();+ long[] words = new long[numWords];+ for (int i = 0; i < numWords; i++) {+ words[i] = sharedBits.get(i);+ }+ return new LongBitSet(words, valueCount);+ }++ private void setGlobalOrdBit(long globalOrd) {+ int wordIndex = (int) (globalOrd >> 6);+ long bit = 1L << globalOrd;+ long prev = sharedBits.get(wordIndex);+ while ((prev & bit) == 0) {+ if (sharedBits.compareAndSet(wordIndex, prev, prev | bit)) {+ break;+ }+ prev = sharedBits.get(wordIndex);+ }+ }+}diff --git a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.javaindex df9271fd0fc..756d5916bf0 100644--- a/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java+++ b/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java@@ -572,11 +572,10 @@ public final class JoinUtil {break;case None:if (min <= 1 && max == Integer.MAX_VALUE) {- GlobalOrdinalsCollector globalOrdinalsCollector =- new GlobalOrdinalsCollector(joinField, ordinalMap, valueCount);- searcher.search(rewrittenFromQuery, globalOrdinalsCollector);return new GlobalOrdinalsQuery(- globalOrdinalsCollector.getCollectorOrdinals(),+ searcher.search(+ rewrittenFromQuery,+ new GlobalOrdinalsCollectorManager(joinField, ordinalMap, valueCount)),joinField,ordinalMap,rewrittenToQuery,diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.javaindex 6e38a3d3323..1d820ef9cbb 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java@@ -91,6 +91,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.analysis.MockTokenizer;import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.search.FixedBitSetCollector;import org.apache.lucene.tests.search.QueryUtils;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;@@ -283,7 +284,7 @@ public class TestJoinUtil extends LuceneTestCase {doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));w.addDocument(doc);- IndexSearcher indexSearcher = new IndexSearcher(w.getReader());+ IndexSearcher indexSearcher = newSearcher(w.getReader());w.close();IndexReader r = indexSearcher.getIndexReader();@@ -398,7 +399,7 @@ public class TestJoinUtil extends LuceneTestCase {w.addDocument(doc);IndexReader r = DirectoryReader.open(w);- IndexSearcher indexSearcher = new IndexSearcher(r);+ IndexSearcher indexSearcher = newSearcher(r);SortedDocValues[] values = new SortedDocValues[r.leaves().size()];for (int i = 0; i < values.length; i++) {LeafReader leafReader = r.leaves().get(i).reader();@@ -493,7 +494,7 @@ public class TestJoinUtil extends LuceneTestCase {indexSearcher.search(joinQuery,new MultiCollectorManager(- new BitSetCollectorManager(indexSearcher.getIndexReader().maxDoc()),+ FixedBitSetCollector.createManager(indexSearcher.getIndexReader().maxDoc()),new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE)));final BitSet actualResult = (BitSet) searchResults[0];assertBitSet(expectedResult, actualResult, indexSearcher);@@ -1583,7 +1584,7 @@ public class TestJoinUtil extends LuceneTestCase {indexSearcher.search(joinQuery,new MultiCollectorManager(- new BitSetCollectorManager(indexSearcher.getIndexReader().maxDoc()),+ FixedBitSetCollector.createManager(indexSearcher.getIndexReader().maxDoc()),new TopScoreDocCollectorManager(10, null, Integer.MAX_VALUE)));// Asserting bit set...assertBitSet(expectedResult, (BitSet) searchResults[0], indexSearcher);@@ -2080,31 +2081,6 @@ public class TestJoinUtil extends LuceneTestCase {}}- private static class BitSetCollector extends SimpleCollector {-- private final BitSet bitSet;- private int docBase;-- private BitSetCollector(BitSet bitSet) {- this.bitSet = bitSet;- }-- @Override- public void collect(int doc) throws IOException {- bitSet.set(docBase + doc);- }-- @Override- protected void doSetNextReader(LeafReaderContext context) throws IOException {- docBase = context.docBase;- }-- @Override- public org.apache.lucene.search.ScoreMode scoreMode() {- return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;- }- }-private abstract static class JoinValueCollector extends SimpleCollector {final Map<BytesRef, JoinScore> localMap = new HashMap<>();final String fromField;@@ -2133,8 +2109,12 @@ public class TestJoinUtil extends LuceneTestCase {entry.getKey(),entry.getValue(),(existing, src) -> {- if (src.minScore < existing.minScore) existing.minScore = src.minScore;- if (src.maxScore > existing.maxScore) existing.maxScore = src.maxScore;+ if (src.minScore < existing.minScore) {+ existing.minScore = src.minScore;+ }+ if (src.maxScore > existing.maxScore) {+ existing.maxScore = src.maxScore;+ }existing.total += src.total;existing.count += src.count;return existing;@@ -2250,28 +2230,4 @@ public class TestJoinUtil extends LuceneTestCase {return org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES;}}-- private static class BitSetCollectorManager- implements CollectorManager<BitSetCollector, FixedBitSet> {-- private final int maxDoc;-- BitSetCollectorManager(int maxDoc) {- this.maxDoc = maxDoc;- }-- @Override- public BitSetCollector newCollector() {- return new BitSetCollector(new FixedBitSet(maxDoc));- }-- @Override- public FixedBitSet reduce(Collection<BitSetCollector> collectors) {- FixedBitSet result = new FixedBitSet(maxDoc);- for (BitSetCollector c : collectors) {- result.or((FixedBitSet) c.bitSet);- }- return result;- }- }}