Skip to content

Apache Lucene open-source search software

Reduce memory usage of field maps in FieldInfos and BlockTree TermsReader. (#13327)

Bruno Roustant8c738ba01074
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 22570d8c8e1..5d3adf58643 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -337,6 +337,8 @@ Optimizations
* GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hits
to count. (Greg Miller)
+* GITHUB#13327: Reduce memory usage of field maps in FieldInfos and BlockTree TermsReader. (Bruno Roustant, David Smiley)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
index 753c84f1277..190046e2a3f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
@@ -21,12 +21,12 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
@@ -35,10 +35,11 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.hppc.IntCursor;
+import org.apache.lucene.util.hppc.IntObjectHashMap;
/**
* A block-based terms index and dictionary that assigns terms to variable length blocks according
@@ -113,7 +114,8 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
// produce DocsEnum on demand
final PostingsReaderBase postingsReader;
- private final Map<String, FieldReader> fieldMap;
+ private final FieldInfos fieldInfos;
+ private final IntObjectHashMap<FieldReader> fieldMap;
private final List<String> fieldList;
final String segment;
@@ -157,7 +159,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
// Read per-field details
String metaName =
IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_META_EXTENSION);
- Map<String, FieldReader> fieldMap = null;
+ IntObjectHashMap<FieldReader> fieldMap = null;
Throwable priorE = null;
long indexLength = -1, termsLength = -1;
try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName)) {
@@ -175,7 +177,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
if (numFields < 0) {
throw new CorruptIndexException("invalid numFields: " + numFields, metaIn);
}
- fieldMap = CollectionUtil.newHashMap(numFields);
+ fieldMap = new IntObjectHashMap<>(numFields);
for (int i = 0; i < numFields; ++i) {
final int field = metaIn.readVInt();
final long numTerms = metaIn.readVLong();
@@ -216,7 +218,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
final long indexStartFP = metaIn.readVLong();
FieldReader previous =
fieldMap.put(
- fieldInfo.name,
+ fieldInfo.number,
new FieldReader(
this,
fieldInfo,
@@ -250,10 +252,9 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
// correct
CodecUtil.retrieveChecksum(indexIn, indexLength);
CodecUtil.retrieveChecksum(termsIn, termsLength);
- List<String> fieldList = new ArrayList<>(fieldMap.keySet());
- fieldList.sort(null);
+ fieldInfos = state.fieldInfos;
this.fieldMap = fieldMap;
- this.fieldList = Collections.unmodifiableList(fieldList);
+ this.fieldList = sortFieldNames(fieldMap, state.fieldInfos);
success = true;
} finally {
if (!success) {
@@ -277,6 +278,16 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
return bytes;
}
+ private static List<String> sortFieldNames(
+ IntObjectHashMap<FieldReader> fieldMap, FieldInfos fieldInfos) {
+ List<String> fieldNames = new ArrayList<>(fieldMap.size());
+ for (IntCursor fieldNumberCursor : fieldMap.keys()) {
+ fieldNames.add(fieldInfos.fieldInfo(fieldNumberCursor.value).name);
+ }
+ fieldNames.sort(null);
+ return Collections.unmodifiableList(fieldNames);
+ }
+
// for debugging
// private static String toHex(int v) {
// return "0x" + Integer.toHexString(v);
@@ -301,7 +312,8 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
@Override
public Terms terms(String field) throws IOException {
assert field != null;
- return fieldMap.get(field);
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ return fieldInfo == null ? null : fieldMap.get(fieldInfo.number);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index b212d14f726..108d5a9710d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -23,9 +23,8 @@ import static org.apache.lucene.index.FieldInfo.verifySamePointsOptions;
import static org.apache.lucene.index.FieldInfo.verifySameStoreTermVectors;
import static org.apache.lucene.index.FieldInfo.verifySameVectorOptions;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -34,7 +33,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
-import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.CollectionUtil;
/**
* Collection of {@link FieldInfo}s (accessible by number or by name).
@@ -62,11 +61,15 @@ public class FieldInfos implements Iterable<FieldInfo> {
// used only by fieldInfo(int)
private final FieldInfo[] byNumber;
+ private final HashMap<String, FieldInfo> byName;
- private final HashMap<String, FieldInfo> byName = new HashMap<>();
- private final Collection<FieldInfo> values; // for an unmodifiable iterator
+ /** Iterator in ascending order of field number. */
+ private final Collection<FieldInfo> values;
- /** Constructs a new FieldInfos from an array of FieldInfo objects */
+ /**
+ * Constructs a new FieldInfos from an array of FieldInfo objects. The array can be used directly
+ * as the backing structure.
+ */
public FieldInfos(FieldInfo[] infos) {
boolean hasVectors = false;
boolean hasPostings = false;
@@ -81,30 +84,21 @@ public class FieldInfos implements Iterable<FieldInfo> {
String softDeletesField = null;
String parentField = null;
- int size = 0; // number of elements in byNumberTemp, number of used array slots
- FieldInfo[] byNumberTemp = new FieldInfo[10]; // initial array capacity of 10
+ byName = CollectionUtil.newHashMap(infos.length);
+ int maxFieldNumber = -1;
+ boolean fieldNumberStrictlyAscending = true;
for (FieldInfo info : infos) {
- if (info.number < 0) {
+ int fieldNumber = info.number;
+ if (fieldNumber < 0) {
throw new IllegalArgumentException(
"illegal field number: " + info.number + " for field " + info.name);
}
- size = info.number >= size ? info.number + 1 : size;
- if (info.number >= byNumberTemp.length) { // grow array
- byNumberTemp = ArrayUtil.grow(byNumberTemp, info.number + 1);
- }
- FieldInfo previous = byNumberTemp[info.number];
- if (previous != null) {
- throw new IllegalArgumentException(
- "duplicate field numbers: "
- + previous.name
- + " and "
- + info.name
- + " have: "
- + info.number);
+ if (maxFieldNumber < fieldNumber) {
+ maxFieldNumber = fieldNumber;
+ } else {
+ fieldNumberStrictlyAscending = false;
}
- byNumberTemp[info.number] = info;
-
- previous = byName.put(info.name, info);
+ FieldInfo previous = byName.put(info.name, info);
if (previous != null) {
throw new IllegalArgumentException(
"duplicate field names: "
@@ -156,15 +150,40 @@ public class FieldInfos implements Iterable<FieldInfo> {
this.softDeletesField = softDeletesField;
this.parentField = parentField;
- List<FieldInfo> valuesTemp = new ArrayList<>(infos.length);
- byNumber = new FieldInfo[size];
- for (int i = 0; i < size; i++) {
- byNumber[i] = byNumberTemp[i];
- if (byNumberTemp[i] != null) {
- valuesTemp.add(byNumberTemp[i]);
+ if (fieldNumberStrictlyAscending && maxFieldNumber == infos.length - 1) {
+ // The input FieldInfo[] contains all fields numbered from 0 to infos.length - 1, and they are
+ // sorted, use it directly. This is an optimization when reading a segment with all fields
+ // since the FieldInfo[] is sorted.
+ byNumber = infos;
+ values = Arrays.asList(byNumber);
+ } else {
+ byNumber = new FieldInfo[maxFieldNumber + 1];
+ for (FieldInfo fieldInfo : infos) {
+ FieldInfo existing = byNumber[fieldInfo.number];
+ if (existing != null) {
+ throw new IllegalArgumentException(
+ "duplicate field numbers: "
+ + existing.name
+ + " and "
+ + fieldInfo.name
+ + " have: "
+ + fieldInfo.number);
+ }
+ byNumber[fieldInfo.number] = fieldInfo;
+ }
+ if (maxFieldNumber == infos.length - 1) {
+ // No fields are missing, use byNumber.
+ values = Arrays.asList(byNumber);
+ } else {
+ if (!fieldNumberStrictlyAscending) {
+ // The below code is faster than
+ // Arrays.stream(byNumber).filter(Objects::nonNull).toList(),
+ // mainly when the input FieldInfo[] is small compared to maxFieldNumber.
+ Arrays.sort(infos, (fi1, fi2) -> Integer.compare(fi1.number, fi2.number));
+ }
+ values = Arrays.asList(infos);
}
}
- values = Collections.unmodifiableCollection(valuesTemp);
}
/**
@@ -323,10 +342,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
if (fieldNumber < 0) {
throw new IllegalArgumentException("Illegal field number: " + fieldNumber);
}
- if (fieldNumber >= byNumber.length) {
- return null;
- }
- return byNumber[fieldNumber];
+ return fieldNumber >= byNumber.length ? null : byNumber[fieldNumber];
}
static final class FieldDimensions {
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java b/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java
index 985ed6540d5..5bcf133433f 100644
--- a/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java
+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/StateSet.java
@@ -19,6 +19,7 @@ package org.apache.lucene.util.automaton;
import java.util.Arrays;
import org.apache.lucene.util.hppc.BitMixer;
+import org.apache.lucene.util.hppc.IntCursor;
import org.apache.lucene.util.hppc.IntIntHashMap;
/**
@@ -94,7 +95,7 @@ final class StateSet extends IntSet {
}
arrayCache = new int[inner.size()];
int i = 0;
- for (IntIntHashMap.IntCursor cursor : inner.keys()) {
+ for (IntCursor cursor : inner.keys()) {
arrayCache[i++] = cursor.value;
}
// we need to sort this array since "equals" method depend on this
@@ -114,7 +115,7 @@ final class StateSet extends IntSet {
return hashCode;
}
hashCode = inner.size();
- for (IntIntHashMap.IntCursor cursor : inner.keys()) {
+ for (IntCursor cursor : inner.keys()) {
hashCode += BitMixer.mix(cursor.value);
}
hashUpdated = true;
diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.java b/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.java
new file mode 100644
index 00000000000..147274ccfe6
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/AbstractIterator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util.hppc;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Simplifies the implementation of iterators a bit. Modeled loosely after Google Guava's API.
+ *
+ * <p>Forked from com.carrotsearch.hppc.AbstractIterator
+ */
+public abstract class AbstractIterator<E> implements Iterator<E> {
+ private static final int NOT_CACHED = 0;
+ private static final int CACHED = 1;
+ private static final int AT_END = 2;
+
+ /** Current iterator state. */
+ private int state = NOT_CACHED;
+
+ /** The next element to be returned from {@link #next()} if fetched. */
+ private E nextElement;
+
+ @Override
+ public boolean hasNext() {
+ if (state == NOT_CACHED) {
+ state = CACHED;
+ nextElement = fetch();
+ }
+ return state == CACHED;
+ }
+
+ @Override
+ public E next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ state = NOT_CACHED;
+ return nextElement;
+ }
+
+ /** Default implementation throws {@link UnsupportedOperationException}. */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Fetch next element. The implementation must return {@link #done()} when all elements have been
+ * fetched.
+ *
+ * @return Returns the next value for the iterator or chain-calls {@link #done()}.
+ */
+ protected abstract E fetch();
+
+ /**
+ * Call when done.
+ *
+ * @return Returns a unique sentinel value to indicate end-of-iteration.
+ */
+ protected final E done() {
+ state = AT_END;
+ return null;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.java b/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.java
new file mode 100644
index 00000000000..2c20772b147
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/BufferAllocationException.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util.hppc;
+
+import java.util.IllegalFormatException;
+import java.util.Locale;
+
+/** BufferAllocationException forked from HPPC */
+public class BufferAllocationException extends RuntimeException {
+ public BufferAllocationException(String message) {
+ super(message);
+ }
+
+ public BufferAllocationException(String message, Object... args) {
+ this(message, null, args);
+ }
+
+ public BufferAllocationException(String message, Throwable t, Object... args) {
+ super(formatMessage(message, t, args), t);
+ }
+
+ private static String formatMessage(String message, Throwable t, Object... args) {
+ try {
+ return String.format(Locale.ROOT, message, args);
+ } catch (IllegalFormatException e) {
+ BufferAllocationException substitute =
+ new BufferAllocationException(message + " [ILLEGAL FORMAT, ARGS SUPPRESSED]");
+ if (t != null) {
+ substitute.addSuppressed(t);
+ }
+ substitute.addSuppressed(e);
+ throw substitute;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.java
new file mode 100644
index 00000000000..72a83e0d416
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntCursor.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util.hppc;
+
+/** Forked from HPPC, holding int index and int value */
+public final class IntCursor {
+ /**
+ * The current value's index in the container this cursor belongs to. The meaning of this index is
+ * defined by the container (usually it will be an index in the underlying storage buffer).
+ */
+ public int index;
+
+ /** The current value. */
+ public int value;
+
+ @Override
+ public String toString() {
+ return "[cursor, index: " + index + ", value: " + value + "]";
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java
index 678843960e6..3d37e306a71 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java
@@ -20,10 +20,7 @@ package org.apache.lucene.util.hppc;
import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
import java.util.Arrays;
-import java.util.IllegalFormatException;
import java.util.Iterator;
-import java.util.Locale;
-import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -631,62 +628,6 @@ public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Clon
}
}
- /** Simplifies the implementation of iterators a bit. Modeled loosely after Google Guava's API. */
- public abstract static class AbstractIterator<E> implements Iterator<E> {
- private static final int NOT_CACHED = 0;
- private static final int CACHED = 1;
- private static final int AT_END = 2;
-
- /** Current iterator state. */
- private int state = NOT_CACHED;
-
- /** The next element to be returned from {@link #next()} if fetched. */
- private E nextElement;
-
- @Override
- public boolean hasNext() {
- if (state == NOT_CACHED) {
- state = CACHED;
- nextElement = fetch();
- }
- return state == CACHED;
- }
-
- @Override
- public E next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
-
- state = NOT_CACHED;
- return nextElement;
- }
-
- /** Default implementation throws {@link UnsupportedOperationException}. */
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Fetch next element. The implementation must return {@link #done()} when all elements have
- * been fetched.
- *
- * @return Returns the next value for the iterator or chain-calls {@link #done()}.
- */
- protected abstract E fetch();
-
- /**
- * Call when done.
- *
- * @return Returns a unique sentinel value to indicate end-of-iteration.
- */
- protected final E done() {
- state = AT_END;
- return null;
- }
- }
-
@Override
public IntIntHashMap clone() {
try {
@@ -949,51 +890,4 @@ public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Clon
return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";
}
}
-
- /** Forked from HPPC, holding int index and int value */
- public final class IntCursor {
- /**
- * The current value's index in the container this cursor belongs to. The meaning of this index
- * is defined by the container (usually it will be an index in the underlying storage buffer).
- */
- public int index;
-
- /** The current value. */
- public int value;
-
- @Override
- public String toString() {
- return "[cursor, index: " + index + ", value: " + value + "]";
- }
- }
-
- /** BufferAllocationException forked from HPPC */
- @SuppressWarnings("serial")
- public static class BufferAllocationException extends RuntimeException {
- public BufferAllocationException(String message) {
- super(message);
- }
-
- public BufferAllocationException(String message, Object... args) {
- this(message, null, args);
- }
-
- public BufferAllocationException(String message, Throwable t, Object... args) {
- super(formatMessage(message, t, args), t);
- }
-
- private static String formatMessage(String message, Throwable t, Object... args) {
- try {
- return String.format(Locale.ROOT, message, args);
- } catch (IllegalFormatException e) {
- BufferAllocationException substitute =
- new BufferAllocationException(message + " [ILLEGAL FORMAT, ARGS SUPPRESSED]");
- if (t != null) {
- substitute.addSuppressed(t);
- }
- substitute.addSuppressed(e);
- throw substitute;
- }
- }
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java b/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java
new file mode 100644
index 00000000000..f063a0e7f09
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java
@@ -0,0 +1,889 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util.hppc;
+
+import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A hash map of <code>int</code> to <code>Object</code>, implemented using open addressing with
+ * linear probing for collision resolution.
+ *
+ * <p>Mostly forked and trimmed from com.carrotsearch.hppc.IntObjectHashMap
+ *
+ * <p>github: https://github.com/carrotsearch/hppc release 0.9.0
+ */
+@SuppressWarnings("unchecked")
+public class IntObjectHashMap<VType>
+ implements Iterable<IntObjectHashMap.IntObjectCursor<VType>>, Cloneable {
+
+ public static final int DEFAULT_EXPECTED_ELEMENTS = 4;
+
+ public static final float DEFAULT_LOAD_FACTOR = 0.75f;
+
+ private static final AtomicInteger ITERATION_SEED = new AtomicInteger();
+
+ /** Minimal sane load factor (99 empty slots per 100). */
+ public static final float MIN_LOAD_FACTOR = 1 / 100.0f;
+
+ /** Maximum sane load factor (1 empty slot per 100). */
+ public static final float MAX_LOAD_FACTOR = 99 / 100.0f;
+
+ /** Minimum hash buffer size. */
+ public static final int MIN_HASH_ARRAY_LENGTH = 4;
+
+ /**
+ * Maximum array size for hash containers (power-of-two and still allocable in Java, not a
+ * negative int).
+ */
+ public static final int MAX_HASH_ARRAY_LENGTH = 0x80000000 >>> 1;
+
+ /** The array holding keys. */
+ public int[] keys;
+
+ /** The array holding values. */
+ public Object[] values;
+
+ /**
+ * The number of stored keys (assigned key slots), excluding the special "empty" key, if any (use
+ * {@link #size()} instead).
+ *
+ * @see #size()
+ */
+ protected int assigned;
+
+ /** Mask for slot scans in {@link #keys}. */
+ protected int mask;
+
+ /** Expand (rehash) {@link #keys} when {@link #assigned} hits this value. */
+ protected int resizeAt;
+
+ /** Special treatment for the "empty slot" key marker. */
+ protected boolean hasEmptyKey;
+
+ /** The load factor for {@link #keys}. */
+ protected double loadFactor;
+
+ /** Seed used to ensure the hash iteration order is different from an iteration to another. */
+ protected int iterationSeed;
+
+ /** New instance with sane defaults. */
+ public IntObjectHashMap() {
+ this(DEFAULT_EXPECTED_ELEMENTS);
+ }
+
+ /**
+ * New instance with sane defaults.
+ *
+ * @param expectedElements The expected number of elements guaranteed not to cause buffer
+ * expansion (inclusive).
+ */
+ public IntObjectHashMap(int expectedElements) {
+ this(expectedElements, DEFAULT_LOAD_FACTOR);
+ }
+
+ /**
+ * New instance with the provided defaults.
+ *
+ * @param expectedElements The expected number of elements guaranteed not to cause a rehash
+ * (inclusive).
+ * @param loadFactor The load factor for internal buffers. Insane load factors (zero, full
+ * capacity) are rejected by {@link #verifyLoadFactor(double)}.
+ */
+ public IntObjectHashMap(int expectedElements, double loadFactor) {
+ this.loadFactor = verifyLoadFactor(loadFactor);
+ iterationSeed = ITERATION_SEED.incrementAndGet();
+ ensureCapacity(expectedElements);
+ }
+
+ /** Create a hash map from all key-value pairs of another container. */
+ public IntObjectHashMap(Iterable<? extends IntObjectCursor<? extends VType>> container) {
+ this();
+ putAll(container);
+ }
+
+ public VType put(int key, VType value) {
+ assert assigned < mask + 1;
+
+ final int mask = this.mask;
+ if (((key) == 0)) {
+ hasEmptyKey = true;
+ VType previousValue = (VType) values[mask + 1];
+ values[mask + 1] = value;
+ return previousValue;
+ } else {
+ final int[] keys = this.keys;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ final VType previousValue = (VType) values[slot];
+ values[slot] = value;
+ return previousValue;
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ if (assigned == resizeAt) {
+ allocateThenInsertThenRehash(slot, key, value);
+ } else {
+ keys[slot] = key;
+ values[slot] = value;
+ }
+
+ assigned++;
+ return null;
+ }
+ }
+
+ public int putAll(Iterable<? extends IntObjectCursor<? extends VType>> iterable) {
+ final int count = size();
+ for (IntObjectCursor<? extends VType> c : iterable) {
+ put(c.key, c.value);
+ }
+ return size() - count;
+ }
+
+ /**
+ * <a href="http://trove4j.sourceforge.net">Trove</a>-inspired API method. An equivalent of the
+ * following code:
+ *
+ * <pre>
+ * if (!map.containsKey(key)) map.put(value);
+ * </pre>
+ *
+ * @param key The key of the value to check.
+ * @param value The value to put if <code>key</code> does not exist.
+ * @return <code>true</code> if <code>key</code> did not exist and <code>value</code> was placed
+ * in the map.
+ */
+ public boolean putIfAbsent(int key, VType value) {
+ int keyIndex = indexOf(key);
+ if (!indexExists(keyIndex)) {
+ indexInsert(keyIndex, key, value);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public VType remove(int key) {
+ final int mask = this.mask;
+ if (((key) == 0)) {
+ hasEmptyKey = false;
+ VType previousValue = (VType) values[mask + 1];
+ values[mask + 1] = 0;
+ return previousValue;
+ } else {
+ final int[] keys = this.keys;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ final VType previousValue = (VType) values[slot];
+ shiftConflictingKeys(slot);
+ return previousValue;
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ return null;
+ }
+ }
+
+ public VType get(int key) {
+ if (((key) == 0)) {
+ return hasEmptyKey ? (VType) values[mask + 1] : null;
+ } else {
+ final int[] keys = this.keys;
+ final int mask = this.mask;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ return (VType) values[slot];
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ return null;
+ }
+ }
+
+ public VType getOrDefault(int key, VType defaultValue) {
+ if (((key) == 0)) {
+ return hasEmptyKey ? (VType) values[mask + 1] : defaultValue;
+ } else {
+ final int[] keys = this.keys;
+ final int mask = this.mask;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ return (VType) values[slot];
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ return defaultValue;
+ }
+ }
+
+ public boolean containsKey(int key) {
+ if (((key) == 0)) {
+ return hasEmptyKey;
+ } else {
+ final int[] keys = this.keys;
+ final int mask = this.mask;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ return true;
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ return false;
+ }
+ }
+
+ public int indexOf(int key) {
+ final int mask = this.mask;
+ if (((key) == 0)) {
+ return hasEmptyKey ? mask + 1 : ~(mask + 1);
+ } else {
+ final int[] keys = this.keys;
+ int slot = hashKey(key) & mask;
+
+ int existing;
+ while (!((existing = keys[slot]) == 0)) {
+ if (((existing) == (key))) {
+ return slot;
+ }
+ slot = (slot + 1) & mask;
+ }
+
+ return ~slot;
+ }
+ }
+
+ public boolean indexExists(int index) {
+ assert index < 0 || (index >= 0 && index <= mask) || (index == mask + 1 && hasEmptyKey);
+
+ return index >= 0;
+ }
+
+ public VType indexGet(int index) {
+ assert index >= 0 : "The index must point at an existing key.";
+ assert index <= mask || (index == mask + 1 && hasEmptyKey);
+
+ return (VType) values[index];
+ }
+
+ public VType indexReplace(int index, int newValue) {
+ assert index >= 0 : "The index must point at an existing key.";
+ assert index <= mask || (index == mask + 1 && hasEmptyKey);
+
+ VType previousValue = (VType) values[index];
+ values[index] = newValue;
+ return previousValue;
+ }
+
+ public void indexInsert(int index, int key, VType value) {
+ assert index < 0 : "The index must not point at an existing key.";
+
+ index = ~index;
+ if (((key) == 0)) {
+ assert index == mask + 1;
+ values[index] = value;
+ hasEmptyKey = true;
+ } else {
+ assert ((keys[index]) == 0);
+
+ if (assigned == resizeAt) {
+ allocateThenInsertThenRehash(index, key, value);
+ } else {
+ keys[index] = key;
+ values[index] = value;
+ }
+
+ assigned++;
+ }
+ }
+
+ public VType indexRemove(int index) {
+ assert index >= 0 : "The index must point at an existing key.";
+ assert index <= mask || (index == mask + 1 && hasEmptyKey);
+
+ VType previousValue = (VType) values[index];
+ if (index > mask) {
+ hasEmptyKey = false;
+ values[index] = 0;
+ } else {
+ shiftConflictingKeys(index);
+ }
+ return previousValue;
+ }
+
+ public void clear() {
+ assigned = 0;
+ hasEmptyKey = false;
+
+ Arrays.fill(keys, 0);
+
+ /* */
+ }
+
+ public void release() {
+ assigned = 0;
+ hasEmptyKey = false;
+
+ keys = null;
+ values = null;
+ ensureCapacity(DEFAULT_EXPECTED_ELEMENTS);
+ }
+
+ public int size() {
+ return assigned + (hasEmptyKey ? 1 : 0);
+ }
+
+ public boolean isEmpty() {
+ return size() == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int h = hasEmptyKey ? 0xDEADBEEF : 0;
+ for (IntObjectCursor<VType> c : this) {
+ h += BitMixer.mix(c.key) + BitMixer.mix(c.value);
+ }
+ return h;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj != null && getClass() == obj.getClass() && equalElements(getClass().cast(obj));
+ }
+
+ /** Return true if all keys of some other container exist in this container. */
+ protected boolean equalElements(IntObjectHashMap<?> other) {
+ if (other.size() != size()) {
+ return false;
+ }
+
+ for (IntObjectCursor<?> c : other) {
+ int key = c.key;
+ if (!containsKey(key) || !java.util.Objects.equals(c.value, get(key))) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Ensure this container can hold at least the given number of keys (entries) without resizing its
+ * buffers.
+ *
+ * @param expectedElements The total number of keys, inclusive.
+ */
+ public void ensureCapacity(int expectedElements) {
+ if (expectedElements > resizeAt || keys == null) {
+ final int[] prevKeys = this.keys;
+ final VType[] prevValues = (VType[]) this.values;
+ allocateBuffers(minBufferSize(expectedElements, loadFactor));
+ if (prevKeys != null && !isEmpty()) {
+ rehash(prevKeys, prevValues);
+ }
+ }
+ }
+
+ /**
+ * Provides the next iteration seed used to build the iteration starting slot and offset
+ * increment. This method does not need to be synchronized, what matters is that each thread gets
+ * a sequence of varying seeds.
+ */
+ protected int nextIterationSeed() {
+ return iterationSeed = BitMixer.mixPhi(iterationSeed);
+ }
+
+ @Override
+ public Iterator<IntObjectCursor<VType>> iterator() {
+ return new EntryIterator();
+ }
+
+ /** An iterator implementation for {@link #iterator}. */
+ private final class EntryIterator extends AbstractIterator<IntObjectCursor<VType>> {
+ private final IntObjectCursor<VType> cursor;
+ private final int increment;
+ private int index;
+ private int slot;
+
+ public EntryIterator() {
+ cursor = new IntObjectCursor<VType>();
+ int seed = nextIterationSeed();
+ increment = iterationIncrement(seed);
+ slot = seed & mask;
+ }
+
+ @Override
+ protected IntObjectCursor<VType> fetch() {
+ final int mask = IntObjectHashMap.this.mask;
+ while (index <= mask) {
+ int existing;
+ index++;
+ slot = (slot + increment) & mask;
+ if (!((existing = keys[slot]) == 0)) {
+ cursor.index = slot;
+ cursor.key = existing;
+ cursor.value = (VType) values[slot];
+ return cursor;
+ }
+ }
+
+ if (index == mask + 1 && hasEmptyKey) {
+ cursor.index = index;
+ cursor.key = 0;
+ cursor.value = (VType) values[index++];
+ return cursor;
+ }
+
+ return done();
+ }
+ }
+
+ /** Returns a specialized view of the keys of this associated container. */
+ public KeysContainer keys() {
+ return new KeysContainer();
+ }
+
+ /** A view of the keys inside this hash map. */
+ public final class KeysContainer implements Iterable<IntCursor> {
+
+ @Override
+ public Iterator<IntCursor> iterator() {
+ return new KeysIterator();
+ }
+
+ public int size() {
+ return IntObjectHashMap.this.size();
+ }
+
+ public int[] toArray() {
+ int[] array = new int[size()];
+ int i = 0;
+ for (IntCursor cursor : this) {
+ array[i++] = cursor.value;
+ }
+ return array;
+ }
+ }
+
+ /** An iterator over the set of assigned keys. */
+ private final class KeysIterator extends AbstractIterator<IntCursor> {
+ private final IntCursor cursor;
+ private final int increment;
+ private int index;
+ private int slot;
+
+ public KeysIterator() {
+ cursor = new IntCursor();
+ int seed = nextIterationSeed();
+ increment = iterationIncrement(seed);
+ slot = seed & mask;
+ }
+
+ @Override
+ protected IntCursor fetch() {
+ final int mask = IntObjectHashMap.this.mask;
+ while (index <= mask) {
+ int existing;
+ index++;
+ slot = (slot + increment) & mask;
+ if (!((existing = keys[slot]) == 0)) {
+ cursor.index = slot;
+ cursor.value = existing;
+ return cursor;
+ }
+ }
+
+ if (index == mask + 1 && hasEmptyKey) {
+ cursor.index = index++;
+ cursor.value = 0;
+ return cursor;
+ }
+
+ return done();
+ }
+ }
+
+ /**
+ * @return Returns a container with all values stored in this map.
+ */
+ public ValuesContainer values() {
+ return new ValuesContainer();
+ }
+
+ /** A view over the set of values of this map. */
+ public final class ValuesContainer implements Iterable<ObjectCursor<VType>> {
+
+ @Override
+ public Iterator<ObjectCursor<VType>> iterator() {
+ return new ValuesIterator();
+ }
+
+ public int size() {
+ return IntObjectHashMap.this.size();
+ }
+
+ public VType[] toArray() {
+ VType[] array = (VType[]) new Object[size()];
+ int i = 0;
+ for (ObjectCursor<VType> cursor : this) {
+ array[i++] = cursor.value;
+ }
+ return array;
+ }
+ }
+
+ /** An iterator over the set of assigned values. */
+ private final class ValuesIterator extends AbstractIterator<ObjectCursor<VType>> {
+ private final ObjectCursor<VType> cursor;
+ private final int increment;
+ private int index;
+ private int slot;
+
+ public ValuesIterator() {
+ cursor = new ObjectCursor<>();
+ int seed = nextIterationSeed();
+ increment = iterationIncrement(seed);
+ slot = seed & mask;
+ }
+
+ @Override
+ protected ObjectCursor<VType> fetch() {
+ final int mask = IntObjectHashMap.this.mask;
+ while (index <= mask) {
+ index++;
+ slot = (slot + increment) & mask;
+ if (!((keys[slot]) == 0)) {
+ cursor.index = slot;
+ cursor.value = (VType) values[slot];
+ return cursor;
+ }
+ }
+
+ if (index == mask + 1 && hasEmptyKey) {
+ cursor.index = index;
+ cursor.value = (VType) values[index++];
+ return cursor;
+ }
+
+ return done();
+ }
+ }
+
+ @Override
+ public IntObjectHashMap<VType> clone() {
+ try {
+ /* */
+ IntObjectHashMap<VType> cloned = (IntObjectHashMap<VType>) super.clone();
+ cloned.keys = keys.clone();
+ cloned.values = values.clone();
+ cloned.hasEmptyKey = hasEmptyKey;
+ cloned.iterationSeed = nextIterationSeed();
+ return cloned;
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Convert the contents of this map to a human-friendly string. */
+ @Override
+ public String toString() {
+ final StringBuilder buffer = new StringBuilder();
+ buffer.append("[");
+
+ boolean first = true;
+ for (IntObjectCursor<VType> cursor : this) {
+ if (!first) {
+ buffer.append(", ");
+ }
+ buffer.append(cursor.key);
+ buffer.append("=>");
+ buffer.append(cursor.value);
+ first = false;
+ }
+ buffer.append("]");
+ return buffer.toString();
+ }
+
+ /** Creates a hash map from two index-aligned arrays of key-value pairs. */
+ public static <VType> IntObjectHashMap<VType> from(int[] keys, VType[] values) {
+ if (keys.length != values.length) {
+ throw new IllegalArgumentException(
+ "Arrays of keys and values must have an identical length.");
+ }
+
+ IntObjectHashMap<VType> map = new IntObjectHashMap<>(keys.length);
+ for (int i = 0; i < keys.length; i++) {
+ map.put(keys[i], values[i]);
+ }
+
+ return map;
+ }
+
+ /**
+ * Returns a hash code for the given key.
+ *
+ * <p>The output from this function should evenly distribute keys across the entire integer range.
+ */
+ protected int hashKey(int key) {
+ assert !((key) == 0); // Handled as a special case (empty slot marker).
+ return BitMixer.mixPhi(key);
+ }
+
+ /**
+ * Validate load factor range and return it. Override and suppress if you need insane load
+ * factors.
+ */
+ protected double verifyLoadFactor(double loadFactor) {
+ checkLoadFactor(loadFactor, MIN_LOAD_FACTOR, MAX_LOAD_FACTOR);
+ return loadFactor;
+ }
+
+ /** Rehash from old buffers to new buffers. */
+ protected void rehash(int[] fromKeys, VType[] fromValues) {
+ assert fromKeys.length == fromValues.length && checkPowerOfTwo(fromKeys.length - 1);
+
+ // Rehash all stored key/value pairs into the new buffers.
+ final int[] keys = this.keys;
+ final VType[] values = (VType[]) this.values;
+ final int mask = this.mask;
+ int existing;
+
+ // Copy the zero element's slot, then rehash everything else.
+ int from = fromKeys.length - 1;
+ keys[keys.length - 1] = fromKeys[from];
+ values[values.length - 1] = fromValues[from];
+ while (--from >= 0) {
+ if (!((existing = fromKeys[from]) == 0)) {
+ int slot = hashKey(existing) & mask;
+ while (!((keys[slot]) == 0)) {
+ slot = (slot + 1) & mask;
+ }
+ keys[slot] = existing;
+ values[slot] = fromValues[from];
+ }
+ }
+ }
+
+ /**
+ * Allocate new internal buffers. This method attempts to allocate and assign internal buffers
+ * atomically (either allocations succeed or not).
+ */
+ protected void allocateBuffers(int arraySize) {
+ assert Integer.bitCount(arraySize) == 1;
+
+ // Ensure no change is done if we hit an OOM.
+ int[] prevKeys = this.keys;
+ VType[] prevValues = (VType[]) this.values;
+ try {
+ int emptyElementSlot = 1;
+ this.keys = (new int[arraySize + emptyElementSlot]);
+ this.values = new Object[arraySize + emptyElementSlot];
+ } catch (OutOfMemoryError e) {
+ this.keys = prevKeys;
+ this.values = prevValues;
+ throw new BufferAllocationException(
+ "Not enough memory to allocate buffers for rehashing: %,d -> %,d",
+ e, this.mask + 1, arraySize);
+ }
+
+ this.resizeAt = expandAtCount(arraySize, loadFactor);
+ this.mask = arraySize - 1;
+ }
+
+ /**
+ * This method is invoked when there is a new key/ value pair to be inserted into the buffers but
+ * there is not enough empty slots to do so.
+ *
+ * <p>New buffers are allocated. If this succeeds, we know we can proceed with rehashing so we
+ * assign the pending element to the previous buffer (possibly violating the invariant of having
+ * at least one empty slot) and rehash all keys, substituting new buffers at the end.
+ */
+ protected void allocateThenInsertThenRehash(int slot, int pendingKey, VType pendingValue) {
+ assert assigned == resizeAt && ((keys[slot]) == 0) && !((pendingKey) == 0);
+
+ // Try to allocate new buffers first. If we OOM, we leave in a consistent state.
+ final int[] prevKeys = this.keys;
+ final VType[] prevValues = (VType[]) this.values;
+ allocateBuffers(nextBufferSize(mask + 1, size(), loadFactor));
+ assert this.keys.length > prevKeys.length;
+
+ // We have succeeded at allocating new data so insert the pending key/value at
+ // the free slot in the old arrays before rehashing.
+ prevKeys[slot] = pendingKey;
+ prevValues[slot] = pendingValue;
+
+ // Rehash old keys, including the pending key.
+ rehash(prevKeys, prevValues);
+ }
+
+ static int nextBufferSize(int arraySize, int elements, double loadFactor) {
+ assert checkPowerOfTwo(arraySize);
+ if (arraySize == MAX_HASH_ARRAY_LENGTH) {
+ throw new BufferAllocationException(
+ "Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
+ elements, loadFactor);
+ }
+
+ return arraySize << 1;
+ }
+
+ static int expandAtCount(int arraySize, double loadFactor) {
+ assert checkPowerOfTwo(arraySize);
+ // Take care of hash container invariant (there has to be at least one empty slot to ensure
+ // the lookup loop finds either the element or an empty slot).
+ return Math.min(arraySize - 1, (int) Math.ceil(arraySize * loadFactor));
+ }
+
+ static boolean checkPowerOfTwo(int arraySize) {
+ // These are internals, we can just assert without retrying.
+ assert arraySize > 1;
+ assert nextHighestPowerOfTwo(arraySize) == arraySize;
+ return true;
+ }
+
+ static int minBufferSize(int elements, double loadFactor) {
+ if (elements < 0) {
+ throw new IllegalArgumentException("Number of elements must be >= 0: " + elements);
+ }
+
+ long length = (long) Math.ceil(elements / loadFactor);
+ if (length == elements) {
+ length++;
+ }
+ length = Math.max(MIN_HASH_ARRAY_LENGTH, nextHighestPowerOfTwo(length));
+
+ if (length > MAX_HASH_ARRAY_LENGTH) {
+ throw new BufferAllocationException(
+ "Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
+ elements, loadFactor);
+ }
+
+ return (int) length;
+ }
+
+ static void checkLoadFactor(
+ double loadFactor, double minAllowedInclusive, double maxAllowedInclusive) {
+ if (loadFactor < minAllowedInclusive || loadFactor > maxAllowedInclusive) {
+ throw new BufferAllocationException(
+ "The load factor should be in range [%.2f, %.2f]: %f",
+ minAllowedInclusive, maxAllowedInclusive, loadFactor);
+ }
+ }
+
+ static int iterationIncrement(int seed) {
+ return 29 + ((seed & 7) << 1); // Small odd integer.
+ }
+
+ /**
+ * Shift all the slot-conflicting keys and values allocated to (and including) <code>slot</code>.
+ */
+ protected void shiftConflictingKeys(int gapSlot) {
+ final int[] keys = this.keys;
+ final VType[] values = (VType[]) this.values;
+ final int mask = this.mask;
+
+ // Perform shifts of conflicting keys to fill in the gap.
+ int distance = 0;
+ while (true) {
+ final int slot = (gapSlot + (++distance)) & mask;
+ final int existing = keys[slot];
+ if (((existing) == 0)) {
+ break;
+ }
+
+ final int idealSlot = hashKey(existing);
+ final int shift = (slot - idealSlot) & mask;
+ if (shift >= distance) {
+ // Entry at this position was originally at or before the gap slot.
+ // Move the conflict-shifted entry to the gap's position and repeat the procedure
+ // for any entries to the right of the current position, treating it
+ // as the new gap.
+ keys[gapSlot] = existing;
+ values[gapSlot] = values[slot];
+ gapSlot = slot;
+ distance = 0;
+ }
+ }
+
+ // Mark the last found gap slot without a conflict as empty.
+ keys[gapSlot] = 0;
+ values[gapSlot] = null;
+ assigned--;
+ }
+
+ /** Forked from HPPC, holding int index,key and value */
+ public static final class IntObjectCursor<VType> {
+ /**
+ * The current key and value's index in the container this cursor belongs to. The meaning of
+ * this index is defined by the container (usually it will be an index in the underlying storage
+ * buffer).
+ */
+ public int index;
+
+ /** The current key. */
+ public int key;
+
+ /** The current value. */
+ public VType value;
+
+ @Override
+ public String toString() {
+ return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";
+ }
+ }
+
+ /** Forked from HPPC, holding int index and Object value */
+ public static final class ObjectCursor<VType> {
+ /**
+ * The current value's index in the container this cursor belongs to. The meaning of this index
+ * is defined by the container (usually it will be an index in the underlying storage buffer).
+ */
+ public int index;
+
+ /** The current value. */
+ public VType value;
+
+ @Override
+ public String toString() {
+ return "[cursor, index: " + index + ", value: " + value + "]";
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java
index a72c6e77e68..84a4ce96533 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java
@@ -405,7 +405,7 @@ public class TestIntIntHashMap extends LuceneTestCase {
map.put(key3, value1);
int counted = 0;
- for (IntIntHashMap.IntCursor c : map.keys()) {
+ for (IntCursor c : map.keys()) {
assertEquals(map.keys[c.index], c.value);
counted++;
}
@@ -581,7 +581,7 @@ public class TestIntIntHashMap extends LuceneTestCase {
map.put(key3, value1);
int counted = 0;
- for (IntIntHashMap.IntCursor c : map.values()) {
+ for (IntCursor c : map.values()) {
assertEquals(map.values[c.index], c.value);
counted++;
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java
new file mode 100644
index 00000000000..dcfea7ab095
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util.hppc;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Tests for {@link IntObjectHashMap}.
+ *
+ * <p>Mostly forked and trimmed from com.carrotsearch.hppc.IntObjectHashMapTest
+ *
+ * <p>github: https://github.com/carrotsearch/hppc release: 0.9.0
+ */
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class TestIntObjectHashMap extends LuceneTestCase {
+ /* Ready to use key values. */
+
+ protected int keyE = 0;
+ protected int key0 = cast(0), k0 = key0;
+ protected int key1 = cast(1), k1 = key1;
+ protected int key2 = cast(2), k2 = key2;
+ protected int key3 = cast(3), k3 = key3;
+ protected int key4 = cast(4), k4 = key4;
+ protected int key5 = cast(5), k5 = key5;
+ protected int key6 = cast(6), k6 = key6;
+ protected int key7 = cast(7), k7 = key7;
+ protected int key8 = cast(8), k8 = key8;
+ protected int key9 = cast(9), k9 = key9;
+
+ /** Convert to target type from an integer used to test stuff. */
+ public int cast(Integer v) {
+ return v.intValue();
+ }
+
+ public int[] asArray(int... ints) {
+ int[] values = (new int[ints.length]);
+ for (int i = 0; i < ints.length; i++) values[i] = ints[i];
+ return values;
+ }
+
+ /** Create a new array of a given type and copy the arguments to this array. */
+ /* */
+ public final int[] newArray(int... elements) {
+ return newArray0(elements);
+ }
+
+ /* */
+ private final int[] newArray0(int... elements) {
+ return elements;
+ }
+
+ public int[] newArray(int v0) {
+ return this.newArray0(v0);
+ }
+
+ public int[] newArray(int v0, int v1) {
+ return this.newArray0(v0, v1);
+ }
+
+ public int[] newArray(int v0, int v1, int v2) {
+ return this.newArray0(v0, v1, v2);
+ }
+
+ public int[] newArray(int v0, int v1, int v2, int v3) {
+ return this.newArray0(v0, v1, v2, v3);
+ }
+
+ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6) {
+ return this.newArray0(v0, v1, v2, v3, v4, v5, v6);
+ }
+
+ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5) {
+ return this.newArray0(v0, v1, v2, v3, v4, v5);
+ }
+
+ public int[] newArray(int v0, int v1, int v2, int v3, int v4) {
+ return this.newArray0(v0, v1, v2, v3, v4);
+ }
+
+ public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7) {
+ return this.newArray0(v0, v1, v2, v3, v4, v5, v6, v7);
+ }
+
+ public static int randomIntBetween(int min, int max) {
+ return min + random().nextInt(max + 1 - min);
+ }
+
+ /** Check if the array's content is identical to a given sequence of elements. */
+ public static void assertSortedListEquals(int[] array, int... elements) {
+ assertEquals(elements.length, array.length);
+ Arrays.sort(array);
+ Arrays.sort(elements);
+ assertArrayEquals(elements, array);
+ }
+
+ /** Check if the array's content is identical to a given sequence of elements. */
+ public static void assertSortedListEquals(Object[] array, Object... elements) {
+ assertEquals(elements.length, array.length);
+ Arrays.sort(array);
+ assertArrayEquals(elements, array);
+ }
+
+ protected int value0 = vcast(0);
+ protected int value1 = vcast(1);
+ protected int value2 = vcast(2);
+ protected int value3 = vcast(3);
+ protected int value4 = vcast(4);
+
+ /** Per-test fresh initialized instance. */
+ public IntObjectHashMap<Object> map = newInstance();
+
+ protected IntObjectHashMap newInstance() {
+ return new IntObjectHashMap();
+ }
+
+ @After
+ public void checkEmptySlotsUninitialized() {
+ if (map != null) {
+ int occupied = 0;
+ for (int i = 0; i <= map.mask; i++) {
+ if (((map.keys[i]) == 0)) {
+
+ } else {
+ occupied++;
+ }
+ }
+ assertEquals(occupied, map.assigned);
+
+ if (!map.hasEmptyKey) {}
+ }
+ }
+
+ /** Convert to target type from an integer used to test stuff. */
+ protected int vcast(int value) {
+ return value;
+ }
+
+ /** Create a new array of a given type and copy the arguments to this array. */
+ /* */
+ protected final Object[] newvArray(Object... elements) {
+ return elements;
+ }
+
+ private void assertSameMap(final IntObjectHashMap<Object> c1, final IntObjectHashMap<Object> c2) {
+ assertEquals(c1.size(), c2.size());
+
+ for (IntObjectHashMap.IntObjectCursor entry : c1) {
+ assertTrue(c2.containsKey(entry.key));
+ assertEquals(entry.value, c2.get(entry.key));
+ }
+ }
+
+ /* */
+ @Test
+ public void testEnsureCapacity() {
+ final AtomicInteger expands = new AtomicInteger();
+ IntObjectHashMap map =
+ new IntObjectHashMap(0) {
+ @Override
+ protected void allocateBuffers(int arraySize) {
+ super.allocateBuffers(arraySize);
+ expands.incrementAndGet();
+ }
+ };
+
+ // Add some elements.
+ final int max = rarely() ? 0 : randomIntBetween(0, 250);
+ for (int i = 0; i < max; i++) {
+ map.put(cast(i), value0);
+ }
+
+ final int additions = randomIntBetween(max, max + 5000);
+ map.ensureCapacity(additions + map.size());
+ final int before = expands.get();
+ for (int i = 0; i < additions; i++) {
+ map.put(cast(i), value0);
+ }
+ assertEquals(before, expands.get());
+ }
+
+ @Test
+ public void testCursorIndexIsValid() {
+ map.put(keyE, value1);
+ map.put(key1, value2);
+ map.put(key2, value3);
+
+ for (IntObjectHashMap.IntObjectCursor c : map) {
+ assertTrue(map.indexExists(c.index));
+ assertEquals(c.value, map.indexGet(c.index));
+ }
+ }
+
+ @Test
+ public void testIndexMethods() {
+ map.put(keyE, value1);
+ map.put(key1, value2);
+
+ assertTrue(map.indexOf(keyE) >= 0);
+ assertTrue(map.indexOf(key1) >= 0);
+ assertTrue(map.indexOf(key2) < 0);
+
+ assertTrue(map.indexExists(map.indexOf(keyE)));
+ assertTrue(map.indexExists(map.indexOf(key1)));
+ assertFalse(map.indexExists(map.indexOf(key2)));
+
+ assertEquals(value1, map.indexGet(map.indexOf(keyE)));
+ assertEquals(value2, map.indexGet(map.indexOf(key1)));
+
+ expectThrows(
+ AssertionError.class,
+ () -> {
+ map.indexGet(map.indexOf(key2));
+ fail();
+ });
+
+ assertEquals(value1, map.indexReplace(map.indexOf(keyE), value3));
+ assertEquals(value2, map.indexReplace(map.indexOf(key1), value4));
+ assertEquals(value3, map.indexGet(map.indexOf(keyE)));
+ assertEquals(value4, map.indexGet(map.indexOf(key1)));
+
+ map.indexInsert(map.indexOf(key2), key2, value1);
+ assertEquals(value1, map.indexGet(map.indexOf(key2)));
+ assertEquals(3, map.size());
+
+ assertEquals(value3, map.indexRemove(map.indexOf(keyE)));
+ assertEquals(2, map.size());
+ assertEquals(value1, map.indexRemove(map.indexOf(key2)));
+ assertEquals(1, map.size());
+ assertTrue(map.indexOf(keyE) < 0);
+ assertTrue(map.indexOf(key1) >= 0);
+ assertTrue(map.indexOf(key2) < 0);
+ }
+
+ /* */
+ @Test
+ public void testCloningConstructor() {
+ map.put(key1, value1);
+ map.put(key2, value2);
+ map.put(key3, value3);
+
+ assertSameMap(map, new IntObjectHashMap(map));
+ }
+
+ /* */
+ @Test
+ public void testFromArrays() {
+ map.put(key1, value1);
+ map.put(key2, value2);
+ map.put(key3, value3);
+
+ IntObjectHashMap map2 =
+ IntObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
+
+ assertSameMap(map, map2);
+ }
+
+ @Test
+ public void testGetOrDefault() {
+ map.put(key2, value2);
+ assertTrue(map.containsKey(key2));
+
+ map.put(key1, value1);
+ assertEquals(value1, map.getOrDefault(key1, value3));
+ assertEquals(value3, map.getOrDefault(key3, value3));
+ map.remove(key1);
+ assertEquals(value3, map.getOrDefault(key1, value3));
+ }
+
+ /* */
+ @Test
+ public void testPut() {
+ map.put(key1, value1);
+
+ assertTrue(map.containsKey(key1));
+ assertEquals(value1, map.get(key1));
+ }
+
+ /* */
+ @Test
+ public void testPutOverExistingKey() {
+ map.put(key1, value1);
+ assertEquals(value1, map.put(key1, value3));
+ assertEquals(value3, map.get(key1));
+ }
+
+ /* */
+ @Test
+ public void testPutWithExpansions() {
+ final int COUNT = 10000;
+ final Random rnd = new Random(random().nextLong());
+ final HashSet<Object> values = new HashSet<Object>();
+
+ for (int i = 0; i < COUNT; i++) {
+ final int v = rnd.nextInt();
+ final boolean hadKey = values.contains(cast(v));
+ values.add(cast(v));
+
+ assertEquals(hadKey, map.containsKey(cast(v)));
+ map.put(cast(v), vcast(v));
+ assertEquals(values.size(), map.size());
+ }
+ assertEquals(values.size(), map.size());
+ }
+
+ /* */
+ @Test
+ public void testPutAll() {
+ map.put(key1, value1);
+ map.put(key2, value1);
+
+ IntObjectHashMap map2 = newInstance();
+
+ map2.put(key2, value2);
+ map2.put(keyE, value1);
+
+ // One new key (keyE).
+ assertEquals(1, map.putAll(map2));
+
+ // Assert the value under key2 has been replaced.
+ assertEquals(value2, map.get(key2));
+
+ // And key3 has been added.
+ assertEquals(value1, map.get(keyE));
+ assertEquals(3, map.size());
+ }
+
+ /* */
+ @Test
+ public void testPutIfAbsent() {
+ assertTrue(map.putIfAbsent(key1, value1));
+ assertFalse(map.putIfAbsent(key1, value2));
+ assertEquals(value1, map.get(key1));
+ }
+
+ /* */
+ @Test
+ public void testRemove() {
+ map.put(key1, value1);
+ assertEquals(value1, map.remove(key1));
+ assertEquals(null, map.remove(key1));
+ assertEquals(0, map.size());
+
+ // These are internals, but perhaps worth asserting too.
+ assertEquals(0, map.assigned);
+ }
+
+ /* */
+ @Test
+ public void testEmptyKey() {
+ final int empty = 0;
+
+ map.put(empty, value1);
+ assertEquals(1, map.size());
+ assertEquals(false, map.isEmpty());
+ assertEquals(value1, map.get(empty));
+ assertEquals(value1, map.getOrDefault(empty, value2));
+ assertEquals(true, map.iterator().hasNext());
+ assertEquals(empty, map.iterator().next().key);
+ assertEquals(value1, map.iterator().next().value);
+
+ map.remove(empty);
+ assertEquals(null, map.get(empty));
+ }
+
+ /* */
+ @Test
+ public void testMapKeySet() {
+ map.put(key1, value3);
+ map.put(key2, value2);
+ map.put(key3, value1);
+
+ assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
+ }
+
+ /* */
+ @Test
+ public void testMapKeySetIterator() {
+ map.put(key1, value3);
+ map.put(key2, value2);
+ map.put(key3, value1);
+
+ int counted = 0;
+ for (IntCursor c : map.keys()) {
+ assertEquals(map.keys[c.index], c.value);
+ counted++;
+ }
+ assertEquals(counted, map.size());
+ }
+
+ /* */
+ @Test
+ public void testClear() {
+ map.put(key1, value1);
+ map.put(key2, value1);
+ map.clear();
+ assertEquals(0, map.size());
+
+ // These are internals, but perhaps worth asserting too.
+ assertEquals(0, map.assigned);
+
+ // Check if the map behaves properly upon subsequent use.
+ testPutWithExpansions();
+ }
+
+ /* */
+ @Test
+ public void testRelease() {
+ map.put(key1, value1);
+ map.put(key2, value1);
+ map.release();
+ assertEquals(0, map.size());
+
+ // These are internals, but perhaps worth asserting too.
+ assertEquals(0, map.assigned);
+
+ // Check if the map behaves properly upon subsequent use.
+ testPutWithExpansions();
+ }
+
+ /* */
+ @Test
+ public void testIterable() {
+ map.put(key1, value1);
+ map.put(key2, value2);
+ map.put(key3, value3);
+ map.remove(key2);
+
+ int count = 0;
+ for (IntObjectHashMap.IntObjectCursor cursor : map) {
+ count++;
+ assertTrue(map.containsKey(cursor.key));
+ assertEquals(cursor.value, map.get(cursor.key));
+
+ assertEquals(cursor.value, map.values[cursor.index]);
+ assertEquals(cursor.key, map.keys[cursor.index]);
+ }
+ assertEquals(count, map.size());
+
+ map.clear();
+ assertFalse(map.iterator().hasNext());
+ }
+
+ /* */
+ @Test
+ public void testBug_HPPC73_FullCapacityGet() {
+ final AtomicInteger reallocations = new AtomicInteger();
+ final int elements = 0x7F;
+ map =
+ new IntObjectHashMap(elements, 1f) {
+ @Override
+ protected double verifyLoadFactor(double loadFactor) {
+ // Skip load factor sanity range checking.
+ return loadFactor;
+ }
+
+ @Override
+ protected void allocateBuffers(int arraySize) {
+ super.allocateBuffers(arraySize);
+ reallocations.incrementAndGet();
+ }
+ };
+
+ int reallocationsBefore = reallocations.get();
+ assertEquals(reallocationsBefore, 1);
+ for (int i = 1; i <= elements; i++) {
+ map.put(cast(i), value1);
+ }
+
+ // Non-existent key.
+ int outOfSet = cast(elements + 1);
+ map.remove(outOfSet);
+ assertFalse(map.containsKey(outOfSet));
+ assertEquals(reallocationsBefore, reallocations.get());
+
+ // Should not expand because we're replacing an existing element.
+ map.put(k1, value2);
+ assertEquals(reallocationsBefore, reallocations.get());
+
+ // Remove from a full map.
+ map.remove(k1);
+ assertEquals(reallocationsBefore, reallocations.get());
+ map.put(k1, value2);
+
+ // Check expand on "last slot of a full map" condition.
+ map.put(outOfSet, value1);
+ assertEquals(reallocationsBefore + 1, reallocations.get());
+ }
+
+ @Test
+ public void testHashCodeEquals() {
+ IntObjectHashMap l0 = newInstance();
+ assertEquals(0, l0.hashCode());
+ assertEquals(l0, newInstance());
+
+ IntObjectHashMap l1 =
+ IntObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
+
+ IntObjectHashMap l2 =
+ IntObjectHashMap.from(newArray(key2, key1, key3), newvArray(value2, value1, value3));
+
+ IntObjectHashMap l3 = IntObjectHashMap.from(newArray(key1, key2), newvArray(value2, value1));
+
+ assertEquals(l1.hashCode(), l2.hashCode());
+ assertEquals(l1, l2);
+
+ assertFalse(l1.equals(l3));
+ assertFalse(l2.equals(l3));
+ }
+
+ @Test
+ public void testBug_HPPC37() {
+ IntObjectHashMap l1 = IntObjectHashMap.from(newArray(key1), newvArray(value1));
+
+ IntObjectHashMap l2 = IntObjectHashMap.from(newArray(key2), newvArray(value1));
+
+ assertFalse(l1.equals(l2));
+ assertFalse(l2.equals(l1));
+ }
+
+ /*
+ *
+ */
+ @Test
+ public void testClone() {
+ this.map.put(key1, value1);
+ this.map.put(key2, value2);
+ this.map.put(key3, value3);
+
+ IntObjectHashMap cloned = map.clone();
+ cloned.remove(key1);
+
+ assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
+ assertSortedListEquals(cloned.keys().toArray(), key2, key3);
+ }
+
+ /* */
+ @Test
+ public void testMapValues() {
+ map.put(key1, value3);
+ map.put(key2, value2);
+ map.put(key3, value1);
+ assertSortedListEquals(map.values().toArray(), value1, value2, value3);
+
+ map.clear();
+ map.put(key1, value1);
+ map.put(key2, value2);
+ map.put(key3, value2);
+ assertSortedListEquals(map.values().toArray(), value1, value2, value2);
+ }
+
+ /* */
+ @Test
+ public void testMapValuesIterator() {
+ map.put(key1, value3);
+ map.put(key2, value2);
+ map.put(key3, value1);
+
+ int counted = 0;
+ for (IntObjectHashMap.ObjectCursor c : map.values()) {
+ assertEquals(map.values[c.index], c.value);
+ counted++;
+ }
+ assertEquals(counted, map.size());
+ }
+
+ /* */
+ @Test
+ public void testEqualsSameClass() {
+ IntObjectHashMap l1 = newInstance();
+ l1.put(k1, value0);
+ l1.put(k2, value1);
+ l1.put(k3, value2);
+
+ IntObjectHashMap l2 = new IntObjectHashMap(l1);
+ l2.putAll(l1);
+
+ IntObjectHashMap l3 = new IntObjectHashMap(l2);
+ l3.putAll(l2);
+ l3.put(k4, value0);
+
+ assertEquals(l2, l1);
+ assertEquals(l2.hashCode(), l1.hashCode());
+ assertNotEquals(l1, l3);
+ }
+
+ /* */
+ @Test
+ public void testEqualsSubClass() {
+ class Sub extends IntObjectHashMap {}
+ ;
+
+ IntObjectHashMap l1 = newInstance();
+ l1.put(k1, value0);
+ l1.put(k2, value1);
+ l1.put(k3, value2);
+
+ IntObjectHashMap l2 = new Sub();
+ l2.putAll(l1);
+ l2.put(k4, value3);
+
+ IntObjectHashMap l3 = new Sub();
+ l3.putAll(l2);
+
+ assertNotEquals(l1, l2);
+ assertEquals(l3.hashCode(), l2.hashCode());
+ assertEquals(l3, l2);
+ }
+}