Apache Lucene open-source search software
Reuse scratch int[] for ordinal translation (#16228)
Tim Brooksaef501aafd9a
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 38b7958904a..da64a61cc8c 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -315,6 +315,7 @@ API Changes* GITHUB#16224 Add TokenStreamColumn for experimental columnar batch indexing. (Tim Brooks)+New Features---------------------@@ -531,6 +532,8 @@ Optimizations* GITHUB#16257: Support JDK 26 in Panama Vectorization Provider. (Chris Hegarty)+* GITHUB#16228: Reuse scratch int[] for ordinal translation. (Tim Brooks)+Bug Fixes---------------------* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequentdiff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex 9f99330fc2f..5e81ddb2b0a 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -1347,13 +1347,15 @@ final class IndexingChain implements Accountable {pf.docValuesWriter = new BinaryDocValuesWriter(fi, bytesUsed);break;case SORTED:- pf.docValuesWriter = new SortedDocValuesWriter(fi, bytesUsed, docValuesBytePool);+ pf.docValuesWriter =+ new SortedDocValuesWriter(fi, bytesUsed, docValuesBytePool, sharedIndexingScratch);break;case SORTED_NUMERIC:pf.docValuesWriter = new SortedNumericDocValuesWriter(fi, bytesUsed);break;case SORTED_SET:- pf.docValuesWriter = new SortedSetDocValuesWriter(fi, bytesUsed, docValuesBytePool);+ pf.docValuesWriter =+ new SortedSetDocValuesWriter(fi, bytesUsed, docValuesBytePool, sharedIndexingScratch);break;default:throw new AssertionError("unrecognized DocValues.Type: " + dvType);diff --git a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.javaindex 542a2acee0b..0dd86500a7f 100644--- a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java+++ b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java@@ -32,8 +32,12 @@ final class SharedIndexingScratch {/** Size in bytes of the shared byte scratch buffer. */static final int BYTES_SCRATCH_SIZE = 4 * 1024;+ /** Number of ints in the shared int scratch buffer (4 KB). */+ static final int INTS_SCRATCH_SIZE = 1024;+private final Counter bytesUsed;private byte[] bytesScratchBuffer;+ private int[] intsScratchBuffer;SharedIndexingScratch(Counter bytesUsed) {this.bytesUsed = bytesUsed;@@ -52,4 +56,18 @@ final class SharedIndexingScratch {}return bytesScratchBuffer;}++ /**+ * Returns the shared int scratch buffer, allocating it on the first call and tracking its RAM via+ * the {@link Counter} supplied at construction.+ *+ * <p>Callers must treat the returned array as transient scratch.+ */+ int[] intsScratch() {+ if (intsScratchBuffer == null) {+ intsScratchBuffer = new int[INTS_SCRATCH_SIZE];+ bytesUsed.addAndGet((long) INTS_SCRATCH_SIZE * Integer.BYTES);+ }+ return intsScratchBuffer;+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.javaindex 23b71abf1c3..714cc7a792f 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java@@ -48,14 +48,17 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {private long bytesUsed; // this currently only tracks differences in 'pending'private final FieldInfo fieldInfo;private int lastDocID = -1;+ private final SharedIndexingScratch scratch;private PackedLongValues finalOrds;private int[] finalSortedValues;private int[] finalOrdMap;- public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool) {+ public SortedDocValuesWriter(+ FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool, SharedIndexingScratch scratch) {this.fieldInfo = fieldInfo;this.iwBytesUsed = iwBytesUsed;+ this.scratch = scratch;hash =new BytesRefHash(pool,@@ -118,8 +121,12 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {* <p>All ordinals must be in {@code [0, dictionary.length)}.*/void addOrdinalTuples(int baseDocID, List<BytesRef> dictionary, OrdinalsTupleCursor cursor) {- int[] ordToHash = new int[dictionary.size()];- Arrays.fill(ordToHash, -1);+ int dictSize = dictionary.size();+ int[] ordToHash =+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE+ ? scratch.intsScratch()+ : new int[dictSize];+ Arrays.fill(ordToHash, 0, dictSize, -1);int batchDocID;while ((batchDocID = cursor.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {int docID = baseDocID + batchDocID;@@ -152,8 +159,12 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {return;}assert firstDocID > lastDocID;- int[] ordToHash = new int[dictionary.size()];- Arrays.fill(ordToHash, -1);+ int dictSize = dictionary.size();+ int[] ordToHash =+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE+ ? scratch.intsScratch()+ : new int[dictSize];+ Arrays.fill(ordToHash, 0, dictSize, -1);int processed = 0;try {while (processed < n) {diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.javaindex 3c6f4a2d5ff..edaa40e5698 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java@@ -55,15 +55,18 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {private int[] currentValues = new int[8];private int currentUpto;private int maxCount;+ private final SharedIndexingScratch scratch;private PackedLongValues finalOrds;private PackedLongValues finalOrdCounts;private int[] finalSortedValues;private int[] finalOrdMap;- SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool) {+ SortedSetDocValuesWriter(+ FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool, SharedIndexingScratch scratch) {this.fieldInfo = fieldInfo;this.iwBytesUsed = iwBytesUsed;+ this.scratch = scratch;hash =new BytesRefHash(pool,@@ -110,8 +113,12 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {* batch-local and are offset by {@code baseDocID} to produce segment-level ids.*/void addOrdinalTuples(int baseDocID, List<BytesRef> dictionary, OrdinalsTupleCursor cursor) {- int[] ordToHash = new int[dictionary.size()];- Arrays.fill(ordToHash, -1);+ int dictSize = dictionary.size();+ int[] ordToHash =+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE+ ? scratch.intsScratch()+ : new int[dictSize];+ Arrays.fill(ordToHash, 0, dictSize, -1);int batchDocID;while ((batchDocID = cursor.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {int docID = baseDocID + batchDocID;