Apache Lucene open-source search software
Reduce memory usage of field maps in FieldInfos and BlockTree TermsReader. (#13327)
Bruno Roustant8c738ba01074
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 22570d8c8e1..5d3adf58643 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -337,6 +337,8 @@ Optimizations* GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hitsto count. (Greg Miller)+* GITHUB#13327: Reduce memory usage of field maps in FieldInfos and BlockTree TermsReader. (Bruno Roustant, David Smiley)+Bug Fixes---------------------diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.javaindex 753c84f1277..190046e2a3f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java@@ -21,12 +21,12 @@ import java.util.ArrayList;import java.util.Collections;import java.util.Iterator;import java.util.List;-import java.util.Map;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.SegmentReadState;@@ -35,10 +35,11 @@ import org.apache.lucene.store.ChecksumIndexInput;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.CollectionUtil;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.fst.ByteSequenceOutputs;import org.apache.lucene.util.fst.Outputs;+import org.apache.lucene.util.hppc.IntCursor;+import org.apache.lucene.util.hppc.IntObjectHashMap;/*** A block-based terms index and dictionary that assigns terms to variable length blocks according@@ -113,7 +114,8 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {// produce DocsEnum on demandfinal PostingsReaderBase postingsReader;- private final Map<String, FieldReader> fieldMap;+ private final FieldInfos fieldInfos;+ private final IntObjectHashMap<FieldReader> fieldMap;private final List<String> fieldList;final String segment;@@ -157,7 +159,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {// Read per-field detailsString metaName =IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_META_EXTENSION);- Map<String, FieldReader> fieldMap = null;+ IntObjectHashMap<FieldReader> fieldMap = null;Throwable priorE = null;long indexLength = -1, termsLength = -1;try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName)) {@@ -175,7 +177,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {if (numFields < 0) {throw new CorruptIndexException("invalid numFields: " + numFields, metaIn);}- fieldMap = CollectionUtil.newHashMap(numFields);+ fieldMap = new IntObjectHashMap<>(numFields);for (int i = 0; i < numFields; ++i) {final int field = metaIn.readVInt();final long numTerms = metaIn.readVLong();@@ -216,7 +218,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {final long indexStartFP = metaIn.readVLong();FieldReader previous =fieldMap.put(- fieldInfo.name,+ fieldInfo.number,new FieldReader(this,fieldInfo,@@ -250,10 +252,9 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {// correctCodecUtil.retrieveChecksum(indexIn, indexLength);CodecUtil.retrieveChecksum(termsIn, termsLength);- List<String> fieldList = new ArrayList<>(fieldMap.keySet());- fieldList.sort(null);+ fieldInfos = state.fieldInfos;this.fieldMap = fieldMap;- this.fieldList = Collections.unmodifiableList(fieldList);+ this.fieldList = sortFieldNames(fieldMap, state.fieldInfos);success = true;} finally {if (!success) {@@ -277,6 +278,16 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {return bytes;}+ private static List<String> sortFieldNames(+ IntObjectHashMap<FieldReader> fieldMap, FieldInfos fieldInfos) {+ List<String> fieldNames = new ArrayList<>(fieldMap.size());+ for (IntCursor fieldNumberCursor : fieldMap.keys()) {+ fieldNames.add(fieldInfos.fieldInfo(fieldNumberCursor.value).name);+ }+ fieldNames.sort(null);+ return Collections.unmodifiableList(fieldNames);+ }+// for debugging// private static String toHex(int v) {// return "0x" + Integer.toHexString(v);@@ -301,7 +312,8 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {@Overridepublic Terms terms(String field) throws IOException {assert field != null;- return fieldMap.get(field);+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);+ return fieldInfo == null ? null : fieldMap.get(fieldInfo.number);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.javaindex b212d14f726..108d5a9710d 100644--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java@@ -23,9 +23,8 @@ import static org.apache.lucene.index.FieldInfo.verifySamePointsOptions;import static org.apache.lucene.index.FieldInfo.verifySameStoreTermVectors;import static org.apache.lucene.index.FieldInfo.verifySameVectorOptions;-import java.util.ArrayList;+import java.util.Arrays;import java.util.Collection;-import java.util.Collections;import java.util.HashMap;import java.util.Iterator;import java.util.List;@@ -34,7 +33,7 @@ import java.util.Objects;import java.util.Set;import java.util.stream.Collectors;import java.util.stream.StreamSupport;-import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.CollectionUtil;/*** Collection of {@link FieldInfo}s (accessible by number or by name).@@ -62,11 +61,15 @@ public class FieldInfos implements Iterable<FieldInfo> {// used only by fieldInfo(int)private final FieldInfo[] byNumber;+ private final HashMap<String, FieldInfo> byName;- private final HashMap<String, FieldInfo> byName = new HashMap<>();- private final Collection<FieldInfo> values; // for an unmodifiable iterator+ /** Iterator in ascending order of field number. */+ private final Collection<FieldInfo> values;- /** Constructs a new FieldInfos from an array of FieldInfo objects */+ /**+ * Constructs a new FieldInfos from an array of FieldInfo objects. The array can be used directly+ * as the backing structure.+ */public FieldInfos(FieldInfo[] infos) {boolean hasVectors = false;boolean hasPostings = false;@@ -81,30 +84,21 @@ public class FieldInfos implements Iterable<FieldInfo> {String softDeletesField = null;String parentField = null;- int size = 0; // number of elements in byNumberTemp, number of used array slots- FieldInfo[] byNumberTemp = new FieldInfo[10]; // initial array capacity of 10+ byName = CollectionUtil.newHashMap(infos.length);+ int maxFieldNumber = -1;+ boolean fieldNumberStrictlyAscending = true;for (FieldInfo info : infos) {- if (info.number < 0) {+ int fieldNumber = info.number;+ if (fieldNumber < 0) {throw new IllegalArgumentException("illegal field number: " + info.number + " for field " + info.name);}- size = info.number >= size ? info.number + 1 : size;- if (info.number >= byNumberTemp.length) { // grow array- byNumberTemp = ArrayUtil.grow(byNumberTemp, info.number + 1);- }- FieldInfo previous = byNumberTemp[info.number];- if (previous != null) {- throw new IllegalArgumentException(- "duplicate field numbers: "- + previous.name- + " and "- + info.name- + " have: "- + info.number);+ if (maxFieldNumber < fieldNumber) {+ maxFieldNumber = fieldNumber;+ } else {+ fieldNumberStrictlyAscending = false;}- byNumberTemp[info.number] = info;-- previous = byName.put(info.name, info);+ FieldInfo previous = byName.put(info.name, info);if (previous != null) {throw new IllegalArgumentException("duplicate field names: "@@ -156,15 +150,40 @@ public class FieldInfos implements Iterable<FieldInfo> {this.softDeletesField = softDeletesField;this.parentField = parentField;- List<FieldInfo> valuesTemp = new ArrayList<>(infos.length);- byNumber = new FieldInfo[size];- for (int i = 0; i < size; i++) {- byNumber[i] = byNumberTemp[i];- if (byNumberTemp[i] != null) {- valuesTemp.add(byNumberTemp[i]);+ if (fieldNumberStrictlyAscending && maxFieldNumber == infos.length - 1) {+ // The input FieldInfo[] contains all fields numbered from 0 to infos.length - 1, and they are+ // sorted, use it directly. This is an optimization when reading a segment with all fields+ // since the FieldInfo[] is sorted.+ byNumber = infos;+ values = Arrays.asList(byNumber);+ } else {+ byNumber = new FieldInfo[maxFieldNumber + 1];+ for (FieldInfo fieldInfo : infos) {+ FieldInfo existing = byNumber[fieldInfo.number];+ if (existing != null) {+ throw new IllegalArgumentException(+ "duplicate field numbers: "+ + existing.name+ + " and "+ + fieldInfo.name+ + " have: "+ + fieldInfo.number);+ }+ byNumber[fieldInfo.number] = fieldInfo;+ }+ if (maxFieldNumber == infos.length - 1) {+ // No fields are missing, use byNumber.+ values = Arrays.asList(byNumber);+ } else {+ if (!fieldNumberStrictlyAscending) {+ // The below code is faster than+ // Arrays.stream(byNumber).filter(Objects::nonNull).toList(),+ // mainly when the input FieldInfo[] is small compared to maxFieldNumber.+ Arrays.sort(infos, (fi1, fi2) -> Integer.compare(fi1.number, fi2.number));+ }+ values = Arrays.asList(infos);}}- values = Collections.unmodifiableCollection(valuesTemp);}/**@@ -323,10 +342,7 @@ public class FieldInfos implements Iterable<FieldInfo> {if (fieldNumber < 0) {throw new IllegalArgumentException("Illegal field number: " + fieldNumber);}- if (fieldNumber >= byNumber.length) {- return null;- }- return byNumber[fieldNumber];+ return fieldNumber >= byNumber.length ? null : byNumber[fieldNumber];}static final class FieldDimensions {diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java b/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.javaindex 985ed6540d5..5bcf133433f 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java@@ -19,6 +19,7 @@ package org.apache.lucene.util.automaton;import java.util.Arrays;import org.apache.lucene.util.hppc.BitMixer;+import org.apache.lucene.util.hppc.IntCursor;import org.apache.lucene.util.hppc.IntIntHashMap;/**@@ -94,7 +95,7 @@ final class StateSet extends IntSet {}arrayCache = new int[inner.size()];int i = 0;- for (IntIntHashMap.IntCursor cursor : inner.keys()) {+ for (IntCursor cursor : inner.keys()) {arrayCache[i++] = cursor.value;}// we need to sort this array since "equals" method depend on this@@ -114,7 +115,7 @@ final class StateSet extends IntSet {return hashCode;}hashCode = inner.size();- for (IntIntHashMap.IntCursor cursor : inner.keys()) {+ for (IntCursor cursor : inner.keys()) {hashCode += BitMixer.mix(cursor.value);}hashUpdated = true;diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.java b/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.javanew file mode 100644index 00000000000..147274ccfe6--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.java@@ -0,0 +1,81 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util.hppc;++import java.util.Iterator;+import java.util.NoSuchElementException;++/**+ * Simplifies the implementation of iterators a bit. Modeled loosely after Google Guava's API.+ *+ * <p>Forked from com.carrotsearch.hppc.AbstractIterator+ */+public abstract class AbstractIterator<E> implements Iterator<E> {+ private static final int NOT_CACHED = 0;+ private static final int CACHED = 1;+ private static final int AT_END = 2;++ /** Current iterator state. */+ private int state = NOT_CACHED;++ /** The next element to be returned from {@link #next()} if fetched. */+ private E nextElement;++ @Override+ public boolean hasNext() {+ if (state == NOT_CACHED) {+ state = CACHED;+ nextElement = fetch();+ }+ return state == CACHED;+ }++ @Override+ public E next() {+ if (!hasNext()) {+ throw new NoSuchElementException();+ }++ state = NOT_CACHED;+ return nextElement;+ }++ /** Default implementation throws {@link UnsupportedOperationException}. */+ @Override+ public void remove() {+ throw new UnsupportedOperationException();+ }++ /**+ * Fetch next element. The implementation must return {@link #done()} when all elements have been+ * fetched.+ *+ * @return Returns the next value for the iterator or chain-calls {@link #done()}.+ */+ protected abstract E fetch();++ /**+ * Call when done.+ *+ * @return Returns a unique sentinel value to indicate end-of-iteration.+ */+ protected final E done() {+ state = AT_END;+ return null;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.java b/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.javanew file mode 100644index 00000000000..2c20772b147--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.java@@ -0,0 +1,50 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util.hppc;++import java.util.IllegalFormatException;+import java.util.Locale;++/** BufferAllocationException forked from HPPC */+public class BufferAllocationException extends RuntimeException {+ public BufferAllocationException(String message) {+ super(message);+ }++ public BufferAllocationException(String message, Object... args) {+ this(message, null, args);+ }++ public BufferAllocationException(String message, Throwable t, Object... args) {+ super(formatMessage(message, t, args), t);+ }++ private static String formatMessage(String message, Throwable t, Object... args) {+ try {+ return String.format(Locale.ROOT, message, args);+ } catch (IllegalFormatException e) {+ BufferAllocationException substitute =+ new BufferAllocationException(message + " [ILLEGAL FORMAT, ARGS SUPPRESSED]");+ if (t != null) {+ substitute.addSuppressed(t);+ }+ substitute.addSuppressed(e);+ throw substitute;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.javanew file mode 100644index 00000000000..72a83e0d416--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.java@@ -0,0 +1,35 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util.hppc;++/** Forked from HPPC, holding int index and int value */+public final class IntCursor {+ /**+ * The current value's index in the container this cursor belongs to. The meaning of this index is+ * defined by the container (usually it will be an index in the underlying storage buffer).+ */+ public int index;++ /** The current value. */+ public int value;++ @Override+ public String toString() {+ return "[cursor, index: " + index + ", value: " + value + "]";+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.javaindex 678843960e6..3d37e306a71 100644--- a/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java@@ -20,10 +20,7 @@ package org.apache.lucene.util.hppc;import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;import java.util.Arrays;-import java.util.IllegalFormatException;import java.util.Iterator;-import java.util.Locale;-import java.util.NoSuchElementException;import java.util.concurrent.atomic.AtomicInteger;/**@@ -631,62 +628,6 @@ public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Clon}}- /** Simplifies the implementation of iterators a bit. Modeled loosely after Google Guava's API. */- public abstract static class AbstractIterator<E> implements Iterator<E> {- private static final int NOT_CACHED = 0;- private static final int CACHED = 1;- private static final int AT_END = 2;-- /** Current iterator state. */- private int state = NOT_CACHED;-- /** The next element to be returned from {@link #next()} if fetched. */- private E nextElement;-- @Override- public boolean hasNext() {- if (state == NOT_CACHED) {- state = CACHED;- nextElement = fetch();- }- return state == CACHED;- }-- @Override- public E next() {- if (!hasNext()) {- throw new NoSuchElementException();- }-- state = NOT_CACHED;- return nextElement;- }-- /** Default implementation throws {@link UnsupportedOperationException}. */- @Override- public void remove() {- throw new UnsupportedOperationException();- }-- /**- * Fetch next element. The implementation must return {@link #done()} when all elements have- * been fetched.- *- * @return Returns the next value for the iterator or chain-calls {@link #done()}.- */- protected abstract E fetch();-- /**- * Call when done.- *- * @return Returns a unique sentinel value to indicate end-of-iteration.- */- protected final E done() {- state = AT_END;- return null;- }- }-@Overridepublic IntIntHashMap clone() {try {@@ -949,51 +890,4 @@ public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Clonreturn "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";}}-- /** Forked from HPPC, holding int index and int value */- public final class IntCursor {- /**- * The current value's index in the container this cursor belongs to. The meaning of this index- * is defined by the container (usually it will be an index in the underlying storage buffer).- */- public int index;-- /** The current value. */- public int value;-- @Override- public String toString() {- return "[cursor, index: " + index + ", value: " + value + "]";- }- }-- /** BufferAllocationException forked from HPPC */- @SuppressWarnings("serial")- public static class BufferAllocationException extends RuntimeException {- public BufferAllocationException(String message) {- super(message);- }-- public BufferAllocationException(String message, Object... args) {- this(message, null, args);- }-- public BufferAllocationException(String message, Throwable t, Object... args) {- super(formatMessage(message, t, args), t);- }-- private static String formatMessage(String message, Throwable t, Object... args) {- try {- return String.format(Locale.ROOT, message, args);- } catch (IllegalFormatException e) {- BufferAllocationException substitute =- new BufferAllocationException(message + " [ILLEGAL FORMAT, ARGS SUPPRESSED]");- if (t != null) {- substitute.addSuppressed(t);- }- substitute.addSuppressed(e);- throw substitute;- }- }- }}diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.javanew file mode 100644index 00000000000..f063a0e7f09--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java@@ -0,0 +1,889 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util.hppc;++import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;++import java.util.Arrays;+import java.util.Iterator;+import java.util.concurrent.atomic.AtomicInteger;++/**+ * A hash map of <code>int</code> to <code>Object</code>, implemented using open addressing with+ * linear probing for collision resolution.+ *+ * <p>Mostly forked and trimmed from com.carrotsearch.hppc.IntObjectHashMap+ *+ * <p>github: https://github.com/carrotsearch/hppc release 0.9.0+ */+@SuppressWarnings("unchecked")+public class IntObjectHashMap<VType>+ implements Iterable<IntObjectHashMap.IntObjectCursor<VType>>, Cloneable {++ public static final int DEFAULT_EXPECTED_ELEMENTS = 4;++ public static final float DEFAULT_LOAD_FACTOR = 0.75f;++ private static final AtomicInteger ITERATION_SEED = new AtomicInteger();++ /** Minimal sane load factor (99 empty slots per 100). */+ public static final float MIN_LOAD_FACTOR = 1 / 100.0f;++ /** Maximum sane load factor (1 empty slot per 100). */+ public static final float MAX_LOAD_FACTOR = 99 / 100.0f;++ /** Minimum hash buffer size. */+ public static final int MIN_HASH_ARRAY_LENGTH = 4;++ /**+ * Maximum array size for hash containers (power-of-two and still allocable in Java, not a+ * negative int).+ */+ public static final int MAX_HASH_ARRAY_LENGTH = 0x80000000 >>> 1;++ /** The array holding keys. */+ public int[] keys;++ /** The array holding values. */+ public Object[] values;++ /**+ * The number of stored keys (assigned key slots), excluding the special "empty" key, if any (use+ * {@link #size()} instead).+ *+ * @see #size()+ */+ protected int assigned;++ /** Mask for slot scans in {@link #keys}. */+ protected int mask;++ /** Expand (rehash) {@link #keys} when {@link #assigned} hits this value. */+ protected int resizeAt;++ /** Special treatment for the "empty slot" key marker. */+ protected boolean hasEmptyKey;++ /** The load factor for {@link #keys}. */+ protected double loadFactor;++ /** Seed used to ensure the hash iteration order is different from an iteration to another. */+ protected int iterationSeed;++ /** New instance with sane defaults. */+ public IntObjectHashMap() {+ this(DEFAULT_EXPECTED_ELEMENTS);+ }++ /**+ * New instance with sane defaults.+ *+ * @param expectedElements The expected number of elements guaranteed not to cause buffer+ * expansion (inclusive).+ */+ public IntObjectHashMap(int expectedElements) {+ this(expectedElements, DEFAULT_LOAD_FACTOR);+ }++ /**+ * New instance with the provided defaults.+ *+ * @param expectedElements The expected number of elements guaranteed not to cause a rehash+ * (inclusive).+ * @param loadFactor The load factor for internal buffers. Insane load factors (zero, full+ * capacity) are rejected by {@link #verifyLoadFactor(double)}.+ */+ public IntObjectHashMap(int expectedElements, double loadFactor) {+ this.loadFactor = verifyLoadFactor(loadFactor);+ iterationSeed = ITERATION_SEED.incrementAndGet();+ ensureCapacity(expectedElements);+ }++ /** Create a hash map from all key-value pairs of another container. */+ public IntObjectHashMap(Iterable<? extends IntObjectCursor<? extends VType>> container) {+ this();+ putAll(container);+ }++ public VType put(int key, VType value) {+ assert assigned < mask + 1;++ final int mask = this.mask;+ if (((key) == 0)) {+ hasEmptyKey = true;+ VType previousValue = (VType) values[mask + 1];+ values[mask + 1] = value;+ return previousValue;+ } else {+ final int[] keys = this.keys;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ final VType previousValue = (VType) values[slot];+ values[slot] = value;+ return previousValue;+ }+ slot = (slot + 1) & mask;+ }++ if (assigned == resizeAt) {+ allocateThenInsertThenRehash(slot, key, value);+ } else {+ keys[slot] = key;+ values[slot] = value;+ }++ assigned++;+ return null;+ }+ }++ public int putAll(Iterable<? extends IntObjectCursor<? extends VType>> iterable) {+ final int count = size();+ for (IntObjectCursor<? extends VType> c : iterable) {+ put(c.key, c.value);+ }+ return size() - count;+ }++ /**+ * <a href="http://trove4j.sourceforge.net">Trove</a>-inspired API method. An equivalent of the+ * following code:+ *+ * <pre>+ * if (!map.containsKey(key)) map.put(value);+ * </pre>+ *+ * @param key The key of the value to check.+ * @param value The value to put if <code>key</code> does not exist.+ * @return <code>true</code> if <code>key</code> did not exist and <code>value</code> was placed+ * in the map.+ */+ public boolean putIfAbsent(int key, VType value) {+ int keyIndex = indexOf(key);+ if (!indexExists(keyIndex)) {+ indexInsert(keyIndex, key, value);+ return true;+ } else {+ return false;+ }+ }++ public VType remove(int key) {+ final int mask = this.mask;+ if (((key) == 0)) {+ hasEmptyKey = false;+ VType previousValue = (VType) values[mask + 1];+ values[mask + 1] = 0;+ return previousValue;+ } else {+ final int[] keys = this.keys;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ final VType previousValue = (VType) values[slot];+ shiftConflictingKeys(slot);+ return previousValue;+ }+ slot = (slot + 1) & mask;+ }++ return null;+ }+ }++ public VType get(int key) {+ if (((key) == 0)) {+ return hasEmptyKey ? (VType) values[mask + 1] : null;+ } else {+ final int[] keys = this.keys;+ final int mask = this.mask;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ return (VType) values[slot];+ }+ slot = (slot + 1) & mask;+ }++ return null;+ }+ }++ public VType getOrDefault(int key, VType defaultValue) {+ if (((key) == 0)) {+ return hasEmptyKey ? (VType) values[mask + 1] : defaultValue;+ } else {+ final int[] keys = this.keys;+ final int mask = this.mask;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ return (VType) values[slot];+ }+ slot = (slot + 1) & mask;+ }++ return defaultValue;+ }+ }++ public boolean containsKey(int key) {+ if (((key) == 0)) {+ return hasEmptyKey;+ } else {+ final int[] keys = this.keys;+ final int mask = this.mask;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ return true;+ }+ slot = (slot + 1) & mask;+ }++ return false;+ }+ }++ public int indexOf(int key) {+ final int mask = this.mask;+ if (((key) == 0)) {+ return hasEmptyKey ? mask + 1 : ~(mask + 1);+ } else {+ final int[] keys = this.keys;+ int slot = hashKey(key) & mask;++ int existing;+ while (!((existing = keys[slot]) == 0)) {+ if (((existing) == (key))) {+ return slot;+ }+ slot = (slot + 1) & mask;+ }++ return ~slot;+ }+ }++ public boolean indexExists(int index) {+ assert index < 0 || (index >= 0 && index <= mask) || (index == mask + 1 && hasEmptyKey);++ return index >= 0;+ }++ public VType indexGet(int index) {+ assert index >= 0 : "The index must point at an existing key.";+ assert index <= mask || (index == mask + 1 && hasEmptyKey);++ return (VType) values[index];+ }++ public VType indexReplace(int index, int newValue) {+ assert index >= 0 : "The index must point at an existing key.";+ assert index <= mask || (index == mask + 1 && hasEmptyKey);++ VType previousValue = (VType) values[index];+ values[index] = newValue;+ return previousValue;+ }++ public void indexInsert(int index, int key, VType value) {+ assert index < 0 : "The index must not point at an existing key.";++ index = ~index;+ if (((key) == 0)) {+ assert index == mask + 1;+ values[index] = value;+ hasEmptyKey = true;+ } else {+ assert ((keys[index]) == 0);++ if (assigned == resizeAt) {+ allocateThenInsertThenRehash(index, key, value);+ } else {+ keys[index] = key;+ values[index] = value;+ }++ assigned++;+ }+ }++ public VType indexRemove(int index) {+ assert index >= 0 : "The index must point at an existing key.";+ assert index <= mask || (index == mask + 1 && hasEmptyKey);++ VType previousValue = (VType) values[index];+ if (index > mask) {+ hasEmptyKey = false;+ values[index] = 0;+ } else {+ shiftConflictingKeys(index);+ }+ return previousValue;+ }++ public void clear() {+ assigned = 0;+ hasEmptyKey = false;++ Arrays.fill(keys, 0);++ /* */+ }++ public void release() {+ assigned = 0;+ hasEmptyKey = false;++ keys = null;+ values = null;+ ensureCapacity(DEFAULT_EXPECTED_ELEMENTS);+ }++ public int size() {+ return assigned + (hasEmptyKey ? 1 : 0);+ }++ public boolean isEmpty() {+ return size() == 0;+ }++ @Override+ public int hashCode() {+ int h = hasEmptyKey ? 0xDEADBEEF : 0;+ for (IntObjectCursor<VType> c : this) {+ h += BitMixer.mix(c.key) + BitMixer.mix(c.value);+ }+ return h;+ }++ @Override+ public boolean equals(Object obj) {+ return obj != null && getClass() == obj.getClass() && equalElements(getClass().cast(obj));+ }++ /** Return true if all keys of some other container exist in this container. */+ protected boolean equalElements(IntObjectHashMap<?> other) {+ if (other.size() != size()) {+ return false;+ }++ for (IntObjectCursor<?> c : other) {+ int key = c.key;+ if (!containsKey(key) || !java.util.Objects.equals(c.value, get(key))) {+ return false;+ }+ }++ return true;+ }++ /**+ * Ensure this container can hold at least the given number of keys (entries) without resizing its+ * buffers.+ *+ * @param expectedElements The total number of keys, inclusive.+ */+ public void ensureCapacity(int expectedElements) {+ if (expectedElements > resizeAt || keys == null) {+ final int[] prevKeys = this.keys;+ final VType[] prevValues = (VType[]) this.values;+ allocateBuffers(minBufferSize(expectedElements, loadFactor));+ if (prevKeys != null && !isEmpty()) {+ rehash(prevKeys, prevValues);+ }+ }+ }++ /**+ * Provides the next iteration seed used to build the iteration starting slot and offset+ * increment. This method does not need to be synchronized, what matters is that each thread gets+ * a sequence of varying seeds.+ */+ protected int nextIterationSeed() {+ return iterationSeed = BitMixer.mixPhi(iterationSeed);+ }++ @Override+ public Iterator<IntObjectCursor<VType>> iterator() {+ return new EntryIterator();+ }++ /** An iterator implementation for {@link #iterator}. */+ private final class EntryIterator extends AbstractIterator<IntObjectCursor<VType>> {+ private final IntObjectCursor<VType> cursor;+ private final int increment;+ private int index;+ private int slot;++ public EntryIterator() {+ cursor = new IntObjectCursor<VType>();+ int seed = nextIterationSeed();+ increment = iterationIncrement(seed);+ slot = seed & mask;+ }++ @Override+ protected IntObjectCursor<VType> fetch() {+ final int mask = IntObjectHashMap.this.mask;+ while (index <= mask) {+ int existing;+ index++;+ slot = (slot + increment) & mask;+ if (!((existing = keys[slot]) == 0)) {+ cursor.index = slot;+ cursor.key = existing;+ cursor.value = (VType) values[slot];+ return cursor;+ }+ }++ if (index == mask + 1 && hasEmptyKey) {+ cursor.index = index;+ cursor.key = 0;+ cursor.value = (VType) values[index++];+ return cursor;+ }++ return done();+ }+ }++ /** Returns a specialized view of the keys of this associated container. */+ public KeysContainer keys() {+ return new KeysContainer();+ }++ /** A view of the keys inside this hash map. */+ public final class KeysContainer implements Iterable<IntCursor> {++ @Override+ public Iterator<IntCursor> iterator() {+ return new KeysIterator();+ }++ public int size() {+ return IntObjectHashMap.this.size();+ }++ public int[] toArray() {+ int[] array = new int[size()];+ int i = 0;+ for (IntCursor cursor : this) {+ array[i++] = cursor.value;+ }+ return array;+ }+ }++ /** An iterator over the set of assigned keys. */+ private final class KeysIterator extends AbstractIterator<IntCursor> {+ private final IntCursor cursor;+ private final int increment;+ private int index;+ private int slot;++ public KeysIterator() {+ cursor = new IntCursor();+ int seed = nextIterationSeed();+ increment = iterationIncrement(seed);+ slot = seed & mask;+ }++ @Override+ protected IntCursor fetch() {+ final int mask = IntObjectHashMap.this.mask;+ while (index <= mask) {+ int existing;+ index++;+ slot = (slot + increment) & mask;+ if (!((existing = keys[slot]) == 0)) {+ cursor.index = slot;+ cursor.value = existing;+ return cursor;+ }+ }++ if (index == mask + 1 && hasEmptyKey) {+ cursor.index = index++;+ cursor.value = 0;+ return cursor;+ }++ return done();+ }+ }++ /**+ * @return Returns a container with all values stored in this map.+ */+ public ValuesContainer values() {+ return new ValuesContainer();+ }++ /** A view over the set of values of this map. */+ public final class ValuesContainer implements Iterable<ObjectCursor<VType>> {++ @Override+ public Iterator<ObjectCursor<VType>> iterator() {+ return new ValuesIterator();+ }++ public int size() {+ return IntObjectHashMap.this.size();+ }++ public VType[] toArray() {+ VType[] array = (VType[]) new Object[size()];+ int i = 0;+ for (ObjectCursor<VType> cursor : this) {+ array[i++] = cursor.value;+ }+ return array;+ }+ }++ /** An iterator over the set of assigned values. */+ private final class ValuesIterator extends AbstractIterator<ObjectCursor<VType>> {+ private final ObjectCursor<VType> cursor;+ private final int increment;+ private int index;+ private int slot;++ public ValuesIterator() {+ cursor = new ObjectCursor<>();+ int seed = nextIterationSeed();+ increment = iterationIncrement(seed);+ slot = seed & mask;+ }++ @Override+ protected ObjectCursor<VType> fetch() {+ final int mask = IntObjectHashMap.this.mask;+ while (index <= mask) {+ index++;+ slot = (slot + increment) & mask;+ if (!((keys[slot]) == 0)) {+ cursor.index = slot;+ cursor.value = (VType) values[slot];+ return cursor;+ }+ }++ if (index == mask + 1 && hasEmptyKey) {+ cursor.index = index;+ cursor.value = (VType) values[index++];+ return cursor;+ }++ return done();+ }+ }++ @Override+ public IntObjectHashMap<VType> clone() {+ try {+ /* */+ IntObjectHashMap<VType> cloned = (IntObjectHashMap<VType>) super.clone();+ cloned.keys = keys.clone();+ cloned.values = values.clone();+ cloned.hasEmptyKey = hasEmptyKey;+ cloned.iterationSeed = nextIterationSeed();+ return cloned;+ } catch (CloneNotSupportedException e) {+ throw new RuntimeException(e);+ }+ }++ /** Convert the contents of this map to a human-friendly string. */+ @Override+ public String toString() {+ final StringBuilder buffer = new StringBuilder();+ buffer.append("[");++ boolean first = true;+ for (IntObjectCursor<VType> cursor : this) {+ if (!first) {+ buffer.append(", ");+ }+ buffer.append(cursor.key);+ buffer.append("=>");+ buffer.append(cursor.value);+ first = false;+ }+ buffer.append("]");+ return buffer.toString();+ }++ /** Creates a hash map from two index-aligned arrays of key-value pairs. */+ public static <VType> IntObjectHashMap<VType> from(int[] keys, VType[] values) {+ if (keys.length != values.length) {+ throw new IllegalArgumentException(+ "Arrays of keys and values must have an identical length.");+ }++ IntObjectHashMap<VType> map = new IntObjectHashMap<>(keys.length);+ for (int i = 0; i < keys.length; i++) {+ map.put(keys[i], values[i]);+ }++ return map;+ }++ /**+ * Returns a hash code for the given key.+ *+ * <p>The output from this function should evenly distribute keys across the entire integer range.+ */+ protected int hashKey(int key) {+ assert !((key) == 0); // Handled as a special case (empty slot marker).+ return BitMixer.mixPhi(key);+ }++ /**+ * Validate load factor range and return it. Override and suppress if you need insane load+ * factors.+ */+ protected double verifyLoadFactor(double loadFactor) {+ checkLoadFactor(loadFactor, MIN_LOAD_FACTOR, MAX_LOAD_FACTOR);+ return loadFactor;+ }++ /** Rehash from old buffers to new buffers. */+ protected void rehash(int[] fromKeys, VType[] fromValues) {+ assert fromKeys.length == fromValues.length && checkPowerOfTwo(fromKeys.length - 1);++ // Rehash all stored key/value pairs into the new buffers.+ final int[] keys = this.keys;+ final VType[] values = (VType[]) this.values;+ final int mask = this.mask;+ int existing;++ // Copy the zero element's slot, then rehash everything else.+ int from = fromKeys.length - 1;+ keys[keys.length - 1] = fromKeys[from];+ values[values.length - 1] = fromValues[from];+ while (--from >= 0) {+ if (!((existing = fromKeys[from]) == 0)) {+ int slot = hashKey(existing) & mask;+ while (!((keys[slot]) == 0)) {+ slot = (slot + 1) & mask;+ }+ keys[slot] = existing;+ values[slot] = fromValues[from];+ }+ }+ }++ /**+ * Allocate new internal buffers. This method attempts to allocate and assign internal buffers+ * atomically (either allocations succeed or not).+ */+ protected void allocateBuffers(int arraySize) {+ assert Integer.bitCount(arraySize) == 1;++ // Ensure no change is done if we hit an OOM.+ int[] prevKeys = this.keys;+ VType[] prevValues = (VType[]) this.values;+ try {+ int emptyElementSlot = 1;+ this.keys = (new int[arraySize + emptyElementSlot]);+ this.values = new Object[arraySize + emptyElementSlot];+ } catch (OutOfMemoryError e) {+ this.keys = prevKeys;+ this.values = prevValues;+ throw new BufferAllocationException(+ "Not enough memory to allocate buffers for rehashing: %,d -> %,d",+ e, this.mask + 1, arraySize);+ }++ this.resizeAt = expandAtCount(arraySize, loadFactor);+ this.mask = arraySize - 1;+ }++ /**+ * This method is invoked when there is a new key/ value pair to be inserted into the buffers but+ * there is not enough empty slots to do so.+ *+ * <p>New buffers are allocated. If this succeeds, we know we can proceed with rehashing so we+ * assign the pending element to the previous buffer (possibly violating the invariant of having+ * at least one empty slot) and rehash all keys, substituting new buffers at the end.+ */+ protected void allocateThenInsertThenRehash(int slot, int pendingKey, VType pendingValue) {+ assert assigned == resizeAt && ((keys[slot]) == 0) && !((pendingKey) == 0);++ // Try to allocate new buffers first. If we OOM, we leave in a consistent state.+ final int[] prevKeys = this.keys;+ final VType[] prevValues = (VType[]) this.values;+ allocateBuffers(nextBufferSize(mask + 1, size(), loadFactor));+ assert this.keys.length > prevKeys.length;++ // We have succeeded at allocating new data so insert the pending key/value at+ // the free slot in the old arrays before rehashing.+ prevKeys[slot] = pendingKey;+ prevValues[slot] = pendingValue;++ // Rehash old keys, including the pending key.+ rehash(prevKeys, prevValues);+ }++ static int nextBufferSize(int arraySize, int elements, double loadFactor) {+ assert checkPowerOfTwo(arraySize);+ if (arraySize == MAX_HASH_ARRAY_LENGTH) {+ throw new BufferAllocationException(+ "Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",+ elements, loadFactor);+ }++ return arraySize << 1;+ }++ static int expandAtCount(int arraySize, double loadFactor) {+ assert checkPowerOfTwo(arraySize);+ // Take care of hash container invariant (there has to be at least one empty slot to ensure+ // the lookup loop finds either the element or an empty slot).+ return Math.min(arraySize - 1, (int) Math.ceil(arraySize * loadFactor));+ }++ static boolean checkPowerOfTwo(int arraySize) {+ // These are internals, we can just assert without retrying.+ assert arraySize > 1;+ assert nextHighestPowerOfTwo(arraySize) == arraySize;+ return true;+ }++ static int minBufferSize(int elements, double loadFactor) {+ if (elements < 0) {+ throw new IllegalArgumentException("Number of elements must be >= 0: " + elements);+ }++ long length = (long) Math.ceil(elements / loadFactor);+ if (length == elements) {+ length++;+ }+ length = Math.max(MIN_HASH_ARRAY_LENGTH, nextHighestPowerOfTwo(length));++ if (length > MAX_HASH_ARRAY_LENGTH) {+ throw new BufferAllocationException(+ "Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",+ elements, loadFactor);+ }++ return (int) length;+ }++ static void checkLoadFactor(+ double loadFactor, double minAllowedInclusive, double maxAllowedInclusive) {+ if (loadFactor < minAllowedInclusive || loadFactor > maxAllowedInclusive) {+ throw new BufferAllocationException(+ "The load factor should be in range [%.2f, %.2f]: %f",+ minAllowedInclusive, maxAllowedInclusive, loadFactor);+ }+ }++ static int iterationIncrement(int seed) {+ return 29 + ((seed & 7) << 1); // Small odd integer.+ }++ /**+ * Shift all the slot-conflicting keys and values allocated to (and including) <code>slot</code>.+ */+ protected void shiftConflictingKeys(int gapSlot) {+ final int[] keys = this.keys;+ final VType[] values = (VType[]) this.values;+ final int mask = this.mask;++ // Perform shifts of conflicting keys to fill in the gap.+ int distance = 0;+ while (true) {+ final int slot = (gapSlot + (++distance)) & mask;+ final int existing = keys[slot];+ if (((existing) == 0)) {+ break;+ }++ final int idealSlot = hashKey(existing);+ final int shift = (slot - idealSlot) & mask;+ if (shift >= distance) {+ // Entry at this position was originally at or before the gap slot.+ // Move the conflict-shifted entry to the gap's position and repeat the procedure+ // for any entries to the right of the current position, treating it+ // as the new gap.+ keys[gapSlot] = existing;+ values[gapSlot] = values[slot];+ gapSlot = slot;+ distance = 0;+ }+ }++ // Mark the last found gap slot without a conflict as empty.+ keys[gapSlot] = 0;+ values[gapSlot] = null;+ assigned--;+ }++ /** Forked from HPPC, holding int index,key and value */+ public static final class IntObjectCursor<VType> {+ /**+ * The current key and value's index in the container this cursor belongs to. The meaning of+ * this index is defined by the container (usually it will be an index in the underlying storage+ * buffer).+ */+ public int index;++ /** The current key. */+ public int key;++ /** The current value. */+ public VType value;++ @Override+ public String toString() {+ return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";+ }+ }++ /** Forked from HPPC, holding int index and Object value */+ public static final class ObjectCursor<VType> {+ /**+ * The current value's index in the container this cursor belongs to. The meaning of this index+ * is defined by the container (usually it will be an index in the underlying storage buffer).+ */+ public int index;++ /** The current value. */+ public VType value;++ @Override+ public String toString() {+ return "[cursor, index: " + index + ", value: " + value + "]";+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.javaindex a72c6e77e68..84a4ce96533 100644--- a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java+++ b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java@@ -405,7 +405,7 @@ public class TestIntIntHashMap extends LuceneTestCase {map.put(key3, value1);int counted = 0;- for (IntIntHashMap.IntCursor c : map.keys()) {+ for (IntCursor c : map.keys()) {assertEquals(map.keys[c.index], c.value);counted++;}@@ -581,7 +581,7 @@ public class TestIntIntHashMap extends LuceneTestCase {map.put(key3, value1);int counted = 0;- for (IntIntHashMap.IntCursor c : map.values()) {+ for (IntCursor c : map.values()) {assertEquals(map.values[c.index], c.value);counted++;}diff --git a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.javanew file mode 100644index 00000000000..dcfea7ab095--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java@@ -0,0 +1,629 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.util.hppc;++import java.util.Arrays;+import java.util.HashSet;+import java.util.Random;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.junit.After;+import org.junit.Test;++/**+ * Tests for {@link IntObjectHashMap}.+ *+ * <p>Mostly forked and trimmed from com.carrotsearch.hppc.IntObjectHashMapTest+ *+ * <p>github: https://github.com/carrotsearch/hppc release: 0.9.0+ */+@SuppressWarnings({"rawtypes", "unchecked"})+public class TestIntObjectHashMap extends LuceneTestCase {+ /* Ready to use key values. */++ protected int keyE = 0;+ protected int key0 = cast(0), k0 = key0;+ protected int key1 = cast(1), k1 = key1;+ protected int key2 = cast(2), k2 = key2;+ protected int key3 = cast(3), k3 = key3;+ protected int key4 = cast(4), k4 = key4;+ protected int key5 = cast(5), k5 = key5;+ protected int key6 = cast(6), k6 = key6;+ protected int key7 = cast(7), k7 = key7;+ protected int key8 = cast(8), k8 = key8;+ protected int key9 = cast(9), k9 = key9;++ /** Convert to target type from an integer used to test stuff. */+ public int cast(Integer v) {+ return v.intValue();+ }++ public int[] asArray(int... ints) {+ int[] values = (new int[ints.length]);+ for (int i = 0; i < ints.length; i++) values[i] = ints[i];+ return values;+ }++ /** Create a new array of a given type and copy the arguments to this array. */+ /* */+ public final int[] newArray(int... elements) {+ return newArray0(elements);+ }++ /* */+ private final int[] newArray0(int... elements) {+ return elements;+ }++ public int[] newArray(int v0) {+ return this.newArray0(v0);+ }++ public int[] newArray(int v0, int v1) {+ return this.newArray0(v0, v1);+ }++ public int[] newArray(int v0, int v1, int v2) {+ return this.newArray0(v0, v1, v2);+ }++ public int[] newArray(int v0, int v1, int v2, int v3) {+ return this.newArray0(v0, v1, v2, v3);+ }++ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6) {+ return this.newArray0(v0, v1, v2, v3, v4, v5, v6);+ }++ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5) {+ return this.newArray0(v0, v1, v2, v3, v4, v5);+ }++ public int[] newArray(int v0, int v1, int v2, int v3, int v4) {+ return this.newArray0(v0, v1, v2, v3, v4);+ }++ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7) {+ return this.newArray0(v0, v1, v2, v3, v4, v5, v6, v7);+ }++ public static int randomIntBetween(int min, int max) {+ return min + random().nextInt(max + 1 - min);+ }++ /** Check if the array's content is identical to a given sequence of elements. */+ public static void assertSortedListEquals(int[] array, int... elements) {+ assertEquals(elements.length, array.length);+ Arrays.sort(array);+ Arrays.sort(elements);+ assertArrayEquals(elements, array);+ }++ /** Check if the array's content is identical to a given sequence of elements. */+ public static void assertSortedListEquals(Object[] array, Object... elements) {+ assertEquals(elements.length, array.length);+ Arrays.sort(array);+ assertArrayEquals(elements, array);+ }++ protected int value0 = vcast(0);+ protected int value1 = vcast(1);+ protected int value2 = vcast(2);+ protected int value3 = vcast(3);+ protected int value4 = vcast(4);++ /** Per-test fresh initialized instance. */+ public IntObjectHashMap<Object> map = newInstance();++ protected IntObjectHashMap newInstance() {+ return new IntObjectHashMap();+ }++ @After+ public void checkEmptySlotsUninitialized() {+ if (map != null) {+ int occupied = 0;+ for (int i = 0; i <= map.mask; i++) {+ if (((map.keys[i]) == 0)) {++ } else {+ occupied++;+ }+ }+ assertEquals(occupied, map.assigned);++ if (!map.hasEmptyKey) {}+ }+ }++ /** Convert to target type from an integer used to test stuff. */+ protected int vcast(int value) {+ return value;+ }++ /** Create a new array of a given type and copy the arguments to this array. */+ /* */+ protected final Object[] newvArray(Object... elements) {+ return elements;+ }++ private void assertSameMap(final IntObjectHashMap<Object> c1, final IntObjectHashMap<Object> c2) {+ assertEquals(c1.size(), c2.size());++ for (IntObjectHashMap.IntObjectCursor entry : c1) {+ assertTrue(c2.containsKey(entry.key));+ assertEquals(entry.value, c2.get(entry.key));+ }+ }++ /* */+ @Test+ public void testEnsureCapacity() {+ final AtomicInteger expands = new AtomicInteger();+ IntObjectHashMap map =+ new IntObjectHashMap(0) {+ @Override+ protected void allocateBuffers(int arraySize) {+ super.allocateBuffers(arraySize);+ expands.incrementAndGet();+ }+ };++ // Add some elements.+ final int max = rarely() ? 0 : randomIntBetween(0, 250);+ for (int i = 0; i < max; i++) {+ map.put(cast(i), value0);+ }++ final int additions = randomIntBetween(max, max + 5000);+ map.ensureCapacity(additions + map.size());+ final int before = expands.get();+ for (int i = 0; i < additions; i++) {+ map.put(cast(i), value0);+ }+ assertEquals(before, expands.get());+ }++ @Test+ public void testCursorIndexIsValid() {+ map.put(keyE, value1);+ map.put(key1, value2);+ map.put(key2, value3);++ for (IntObjectHashMap.IntObjectCursor c : map) {+ assertTrue(map.indexExists(c.index));+ assertEquals(c.value, map.indexGet(c.index));+ }+ }++ @Test+ public void testIndexMethods() {+ map.put(keyE, value1);+ map.put(key1, value2);++ assertTrue(map.indexOf(keyE) >= 0);+ assertTrue(map.indexOf(key1) >= 0);+ assertTrue(map.indexOf(key2) < 0);++ assertTrue(map.indexExists(map.indexOf(keyE)));+ assertTrue(map.indexExists(map.indexOf(key1)));+ assertFalse(map.indexExists(map.indexOf(key2)));++ assertEquals(value1, map.indexGet(map.indexOf(keyE)));+ assertEquals(value2, map.indexGet(map.indexOf(key1)));++ expectThrows(+ AssertionError.class,+ () -> {+ map.indexGet(map.indexOf(key2));+ fail();+ });++ assertEquals(value1, map.indexReplace(map.indexOf(keyE), value3));+ assertEquals(value2, map.indexReplace(map.indexOf(key1), value4));+ assertEquals(value3, map.indexGet(map.indexOf(keyE)));+ assertEquals(value4, map.indexGet(map.indexOf(key1)));++ map.indexInsert(map.indexOf(key2), key2, value1);+ assertEquals(value1, map.indexGet(map.indexOf(key2)));+ assertEquals(3, map.size());++ assertEquals(value3, map.indexRemove(map.indexOf(keyE)));+ assertEquals(2, map.size());+ assertEquals(value1, map.indexRemove(map.indexOf(key2)));+ assertEquals(1, map.size());+ assertTrue(map.indexOf(keyE) < 0);+ assertTrue(map.indexOf(key1) >= 0);+ assertTrue(map.indexOf(key2) < 0);+ }++ /* */+ @Test+ public void testCloningConstructor() {+ map.put(key1, value1);+ map.put(key2, value2);+ map.put(key3, value3);++ assertSameMap(map, new IntObjectHashMap(map));+ }++ /* */+ @Test+ public void testFromArrays() {+ map.put(key1, value1);+ map.put(key2, value2);+ map.put(key3, value3);++ IntObjectHashMap map2 =+ IntObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));++ assertSameMap(map, map2);+ }++ @Test+ public void testGetOrDefault() {+ map.put(key2, value2);+ assertTrue(map.containsKey(key2));++ map.put(key1, value1);+ assertEquals(value1, map.getOrDefault(key1, value3));+ assertEquals(value3, map.getOrDefault(key3, value3));+ map.remove(key1);+ assertEquals(value3, map.getOrDefault(key1, value3));+ }++ /* */+ @Test+ public void testPut() {+ map.put(key1, value1);++ assertTrue(map.containsKey(key1));+ assertEquals(value1, map.get(key1));+ }++ /* */+ @Test+ public void testPutOverExistingKey() {+ map.put(key1, value1);+ assertEquals(value1, map.put(key1, value3));+ assertEquals(value3, map.get(key1));+ }++ /* */+ @Test+ public void testPutWithExpansions() {+ final int COUNT = 10000;+ final Random rnd = new Random(random().nextLong());+ final HashSet<Object> values = new HashSet<Object>();++ for (int i = 0; i < COUNT; i++) {+ final int v = rnd.nextInt();+ final boolean hadKey = values.contains(cast(v));+ values.add(cast(v));++ assertEquals(hadKey, map.containsKey(cast(v)));+ map.put(cast(v), vcast(v));+ assertEquals(values.size(), map.size());+ }+ assertEquals(values.size(), map.size());+ }++ /* */+ @Test+ public void testPutAll() {+ map.put(key1, value1);+ map.put(key2, value1);++ IntObjectHashMap map2 = newInstance();++ map2.put(key2, value2);+ map2.put(keyE, value1);++ // One new key (keyE).+ assertEquals(1, map.putAll(map2));++ // Assert the value under key2 has been replaced.+ assertEquals(value2, map.get(key2));++ // And key3 has been added.+ assertEquals(value1, map.get(keyE));+ assertEquals(3, map.size());+ }++ /* */+ @Test+ public void testPutIfAbsent() {+ assertTrue(map.putIfAbsent(key1, value1));+ assertFalse(map.putIfAbsent(key1, value2));+ assertEquals(value1, map.get(key1));+ }++ /* */+ @Test+ public void testRemove() {+ map.put(key1, value1);+ assertEquals(value1, map.remove(key1));+ assertEquals(null, map.remove(key1));+ assertEquals(0, map.size());++ // These are internals, but perhaps worth asserting too.+ assertEquals(0, map.assigned);+ }++ /* */+ @Test+ public void testEmptyKey() {+ final int empty = 0;++ map.put(empty, value1);+ assertEquals(1, map.size());+ assertEquals(false, map.isEmpty());+ assertEquals(value1, map.get(empty));+ assertEquals(value1, map.getOrDefault(empty, value2));+ assertEquals(true, map.iterator().hasNext());+ assertEquals(empty, map.iterator().next().key);+ assertEquals(value1, map.iterator().next().value);++ map.remove(empty);+ assertEquals(null, map.get(empty));+ }++ /* */+ @Test+ public void testMapKeySet() {+ map.put(key1, value3);+ map.put(key2, value2);+ map.put(key3, value1);++ assertSortedListEquals(map.keys().toArray(), key1, key2, key3);+ }++ /* */+ @Test+ public void testMapKeySetIterator() {+ map.put(key1, value3);+ map.put(key2, value2);+ map.put(key3, value1);++ int counted = 0;+ for (IntCursor c : map.keys()) {+ assertEquals(map.keys[c.index], c.value);+ counted++;+ }+ assertEquals(counted, map.size());+ }++ /* */+ @Test+ public void testClear() {+ map.put(key1, value1);+ map.put(key2, value1);+ map.clear();+ assertEquals(0, map.size());++ // These are internals, but perhaps worth asserting too.+ assertEquals(0, map.assigned);++ // Check if the map behaves properly upon subsequent use.+ testPutWithExpansions();+ }++ /* */+ @Test+ public void testRelease() {+ map.put(key1, value1);+ map.put(key2, value1);+ map.release();+ assertEquals(0, map.size());++ // These are internals, but perhaps worth asserting too.+ assertEquals(0, map.assigned);++ // Check if the map behaves properly upon subsequent use.+ testPutWithExpansions();+ }++ /* */+ @Test+ public void testIterable() {+ map.put(key1, value1);+ map.put(key2, value2);+ map.put(key3, value3);+ map.remove(key2);++ int count = 0;+ for (IntObjectHashMap.IntObjectCursor cursor : map) {+ count++;+ assertTrue(map.containsKey(cursor.key));+ assertEquals(cursor.value, map.get(cursor.key));++ assertEquals(cursor.value, map.values[cursor.index]);+ assertEquals(cursor.key, map.keys[cursor.index]);+ }+ assertEquals(count, map.size());++ map.clear();+ assertFalse(map.iterator().hasNext());+ }++ /* */+ @Test+ public void testBug_HPPC73_FullCapacityGet() {+ final AtomicInteger reallocations = new AtomicInteger();+ final int elements = 0x7F;+ map =+ new IntObjectHashMap(elements, 1f) {+ @Override+ protected double verifyLoadFactor(double loadFactor) {+ // Skip load factor sanity range checking.+ return loadFactor;+ }++ @Override+ protected void allocateBuffers(int arraySize) {+ super.allocateBuffers(arraySize);+ reallocations.incrementAndGet();+ }+ };++ int reallocationsBefore = reallocations.get();+ assertEquals(reallocationsBefore, 1);+ for (int i = 1; i <= elements; i++) {+ map.put(cast(i), value1);+ }++ // Non-existent key.+ int outOfSet = cast(elements + 1);+ map.remove(outOfSet);+ assertFalse(map.containsKey(outOfSet));+ assertEquals(reallocationsBefore, reallocations.get());++ // Should not expand because we're replacing an existing element.+ map.put(k1, value2);+ assertEquals(reallocationsBefore, reallocations.get());++ // Remove from a full map.+ map.remove(k1);+ assertEquals(reallocationsBefore, reallocations.get());+ map.put(k1, value2);++ // Check expand on "last slot of a full map" condition.+ map.put(outOfSet, value1);+ assertEquals(reallocationsBefore + 1, reallocations.get());+ }++ @Test+ public void testHashCodeEquals() {+ IntObjectHashMap l0 = newInstance();+ assertEquals(0, l0.hashCode());+ assertEquals(l0, newInstance());++ IntObjectHashMap l1 =+ IntObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));++ IntObjectHashMap l2 =+ IntObjectHashMap.from(newArray(key2, key1, key3), newvArray(value2, value1, value3));++ IntObjectHashMap l3 = IntObjectHashMap.from(newArray(key1, key2), newvArray(value2, value1));++ assertEquals(l1.hashCode(), l2.hashCode());+ assertEquals(l1, l2);++ assertFalse(l1.equals(l3));+ assertFalse(l2.equals(l3));+ }++ @Test+ public void testBug_HPPC37() {+ IntObjectHashMap l1 = IntObjectHashMap.from(newArray(key1), newvArray(value1));++ IntObjectHashMap l2 = IntObjectHashMap.from(newArray(key2), newvArray(value1));++ assertFalse(l1.equals(l2));+ assertFalse(l2.equals(l1));+ }++ /*+ *+ */+ @Test+ public void testClone() {+ this.map.put(key1, value1);+ this.map.put(key2, value2);+ this.map.put(key3, value3);++ IntObjectHashMap cloned = map.clone();+ cloned.remove(key1);++ assertSortedListEquals(map.keys().toArray(), key1, key2, key3);+ assertSortedListEquals(cloned.keys().toArray(), key2, key3);+ }++ /* */+ @Test+ public void testMapValues() {+ map.put(key1, value3);+ map.put(key2, value2);+ map.put(key3, value1);+ assertSortedListEquals(map.values().toArray(), value1, value2, value3);++ map.clear();+ map.put(key1, value1);+ map.put(key2, value2);+ map.put(key3, value2);+ assertSortedListEquals(map.values().toArray(), value1, value2, value2);+ }++ /* */+ @Test+ public void testMapValuesIterator() {+ map.put(key1, value3);+ map.put(key2, value2);+ map.put(key3, value1);++ int counted = 0;+ for (IntObjectHashMap.ObjectCursor c : map.values()) {+ assertEquals(map.values[c.index], c.value);+ counted++;+ }+ assertEquals(counted, map.size());+ }++ /* */+ @Test+ public void testEqualsSameClass() {+ IntObjectHashMap l1 = newInstance();+ l1.put(k1, value0);+ l1.put(k2, value1);+ l1.put(k3, value2);++ IntObjectHashMap l2 = new IntObjectHashMap(l1);+ l2.putAll(l1);++ IntObjectHashMap l3 = new IntObjectHashMap(l2);+ l3.putAll(l2);+ l3.put(k4, value0);++ assertEquals(l2, l1);+ assertEquals(l2.hashCode(), l1.hashCode());+ assertNotEquals(l1, l3);+ }++ /* */+ @Test+ public void testEqualsSubClass() {+ class Sub extends IntObjectHashMap {}+ ;++ IntObjectHashMap l1 = newInstance();+ l1.put(k1, value0);+ l1.put(k2, value1);+ l1.put(k3, value2);++ IntObjectHashMap l2 = new Sub();+ l2.putAll(l1);+ l2.put(k4, value3);++ IntObjectHashMap l3 = new Sub();+ l3.putAll(l2);++ assertNotEquals(l1, l2);+ assertEquals(l3.hashCode(), l2.hashCode());+ assertEquals(l3, l2);+ }+}