Apache Lucene open-source search software
Try using Murmurhash 3 for bloom filters (#12868)
Shubham Chaudhary286b59a2dbbe
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.javaindex 1daa1761fd8..2a0472fa028 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java@@ -54,8 +54,9 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;** <p>A choice of {@link BloomFilterFactory} can be passed to tailor Bloom Filter settings on a* per-field basis. The default configuration is {@link DefaultBloomFilterFactory} which allocates a- * ~8mb bitset and hashes values using {@link MurmurHash64}. This should be suitable for most- * purposes.+ * ~8mb bitset and hashes values using {@link+ * org.apache.lucene.util.StringHelper#murmurhash3_x64_128(BytesRef)}. This should be suitable for+ * most purposes.** <p>The format of the blm file is as follows:*diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.javaindex f1d2dee65c7..7d6fd1b64b5 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java@@ -24,6 +24,7 @@ import org.apache.lucene.util.Accountable;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.StringHelper;/*** A class used to represent a set of many, potentially large, values (e.g. many long strings such@@ -53,7 +54,6 @@ public class FuzzySet implements Accountable {NO};- private HashFunction hashFunction;private FixedBitSet filter;private int bloomSize;private final int hashCount;@@ -138,7 +138,6 @@ public class FuzzySet implements Accountable {super();this.filter = filter;this.bloomSize = bloomSize;- this.hashFunction = MurmurHash64.INSTANCE;this.hashCount = hashCount;}@@ -150,11 +149,12 @@ public class FuzzySet implements Accountable {* @return NO or MAYBE*/public ContainsResult contains(BytesRef value) {- long hash = hashFunction.hash(value);- int msb = (int) (hash >>> Integer.SIZE);- int lsb = (int) hash;+ long[] hash = StringHelper.murmurhash3_x64_128(value);++ long msb = hash[0];+ long lsb = hash[1];for (int i = 0; i < hashCount; i++) {- int bloomPos = (lsb + i * msb);+ int bloomPos = ((int) (lsb + i * msb)) & bloomSize;if (!mayContainValue(bloomPos)) {return ContainsResult.NO;}@@ -216,15 +216,14 @@ public class FuzzySet implements Accountable {* is modulo n'd where n is the chosen size of the internal bitset.** @param value the key value to be hashed- * @throws IOException If there is a low-level I/O error*/- public void addValue(BytesRef value) throws IOException {- long hash = hashFunction.hash(value);- int msb = (int) (hash >>> Integer.SIZE);- int lsb = (int) hash;+ public void addValue(BytesRef value) {+ long[] hash = StringHelper.murmurhash3_x64_128(value);+ long msb = hash[0];+ long lsb = hash[1];for (int i = 0; i < hashCount; i++) {// Bitmasking using bloomSize is effectively a modulo operation.- int bloomPos = (lsb + i * msb) & bloomSize;+ int bloomPos = ((int) (lsb + i * msb)) & bloomSize;filter.set(bloomPos);}}@@ -302,9 +301,7 @@ public class FuzzySet implements Accountable {@Overridepublic String toString() {return getClass().getSimpleName()- + "(hash="- + hashFunction- + ", k="+ + "(k="+ hashCount+ ", bits="+ filter.cardinality()diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.javadeleted file mode 100644index eac514a7bb8..00000000000--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java+++ /dev/null@@ -1,37 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.codecs.bloom;--import org.apache.lucene.util.BytesRef;--/**- * Base class for hashing functions that can be referred to by name. Subclasses are expected to- * provide threadsafe implementations of the hash function on the range of bytes referenced in the- * provided {@link BytesRef}- *- * @lucene.experimental- */-public abstract class HashFunction {-- /**- * Hashes the contents of the referenced bytes- *- * @param bytes the data to be hashed- * @return the hash of the bytes referenced by bytes.offset and length bytes.length- */- public abstract long hash(BytesRef bytes);-}diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.javadeleted file mode 100644index 1d189773143..00000000000--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java+++ /dev/null@@ -1,85 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.codecs.bloom;--import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;--/**- * This is a very fast, non-cryptographic hash suitable for general hash-based lookup. See- * http://murmurhash.googlepages.com/ for more details.- *- * <p>The code from Apache Commons was adapted in the form here to work with BytesRefs with offsets- * and lengths rather than raw byte arrays.- */-public class MurmurHash64 extends HashFunction {- private static final long M64 = 0xc6a4a7935bd1e995L;- private static final int R64 = 47;- public static final HashFunction INSTANCE = new MurmurHash64();-- /**- * Generates a 64-bit hash from byte array of the given length and seed.- *- * @param data The input byte array- * @param seed The initial seed value- * @param length The length of the array- * @return The 64-bit hash of the given array- */- public static long hash64(byte[] data, int seed, int offset, int length) {- long h = (seed & 0xffffffffL) ^ (length * M64);-- final int nblocks = length >> 3;-- // body- for (int i = 0; i < nblocks; i++) {-- long k = (long) BitUtil.VH_LE_LONG.get(data, offset);- k *= M64;- k ^= k >>> R64;- k *= M64;-- h ^= k;- h *= M64;-- offset += Long.BYTES;- }-- int remaining = length & 0x07;- if (0 < remaining) {- for (int i = 0; i < remaining; i++) {- h ^= ((long) data[offset + i] & 0xff) << (Byte.SIZE * i);- }- h *= M64;- }-- h ^= h >>> R64;- h *= M64;- h ^= h >>> R64;-- return h;- }-- @Override- public final long hash(BytesRef br) {- return hash64(br.bytes, 0xe17a1465, br.offset, br.length);- }-- @Override- public String toString() {- return getClass().getSimpleName();- }-}diff --git a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java b/lucene/core/src/java/org/apache/lucene/util/StringHelper.javaindex d264c1da58d..5580f5fbe8e 100644--- a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java+++ b/lucene/core/src/java/org/apache/lucene/util/StringHelper.java@@ -209,6 +209,156 @@ public abstract class StringHelper {return murmurhash3_x86_32(bytes.bytes, bytes.offset, bytes.length, seed);}+ /**+ * Generates 128-bit hash from the byte array with the given offset, length and seed.+ *+ * <p>The code is adopted from Apache Commons (<a+ * href="https://commons.apache.org/proper/commons-codec/jacoco/org.apache.commons.codec.digest/MurmurHash3.java.html">link</a>)+ *+ * @param data The input byte array+ * @param offset The first element of array+ * @param length The length of array+ * @param seed The initial seed value+ * @return The 128-bit hash (2 longs)+ */+ public static long[] murmurhash3_x64_128(+ final byte[] data, final int offset, final int length, final int seed) {+ // Use an unsigned 32-bit integer as the seed+ return murmurhash3_x64_128(data, offset, length, seed & 0xFFFFFFFFL);+ }++ @SuppressWarnings("fallthrough")+ private static long[] murmurhash3_x64_128(+ final byte[] data, final int offset, final int length, final long seed) {+ long h1 = seed;+ long h2 = seed;+ final int nblocks = length >> 4;++ // Constants for 128-bit variant+ final long C1 = 0x87c37b91114253d5L;+ final long C2 = 0x4cf5ad432745937fL;+ final int R1 = 31;+ final int R2 = 27;+ final int R3 = 33;+ final int M = 5;+ final int N1 = 0x52dce729;+ final int N2 = 0x38495ab5;++ // body+ for (int i = 0; i < nblocks; i++) {+ final int index = offset + (i << 4);+ long k1 = (long) BitUtil.VH_LE_LONG.get(data, index);+ long k2 = (long) BitUtil.VH_LE_LONG.get(data, index + 8);++ // mix functions for k1+ k1 *= C1;+ k1 = Long.rotateLeft(k1, R1);+ k1 *= C2;+ h1 ^= k1;+ h1 = Long.rotateLeft(h1, R2);+ h1 += h2;+ h1 = h1 * M + N1;++ // mix functions for k2+ k2 *= C2;+ k2 = Long.rotateLeft(k2, R3);+ k2 *= C1;+ h2 ^= k2;+ h2 = Long.rotateLeft(h2, R1);+ h2 += h1;+ h2 = h2 * M + N2;+ }++ // tail+ long k1 = 0;+ long k2 = 0;+ final int index = offset + (nblocks << 4);+ switch (length & 0x0F) {+ case 15:+ k2 ^= ((long) data[index + 14] & 0xff) << 48;+ case 14:+ k2 ^= ((long) data[index + 13] & 0xff) << 40;+ case 13:+ k2 ^= ((long) data[index + 12] & 0xff) << 32;+ case 12:+ k2 ^= ((long) data[index + 11] & 0xff) << 24;+ case 11:+ k2 ^= ((long) data[index + 10] & 0xff) << 16;+ case 10:+ k2 ^= ((long) data[index + 9] & 0xff) << 8;+ case 9:+ k2 ^= data[index + 8] & 0xff;+ k2 *= C2;+ k2 = Long.rotateLeft(k2, R3);+ k2 *= C1;+ h2 ^= k2;++ case 8:+ k1 ^= ((long) data[index + 7] & 0xff) << 56;+ case 7:+ k1 ^= ((long) data[index + 6] & 0xff) << 48;+ case 6:+ k1 ^= ((long) data[index + 5] & 0xff) << 40;+ case 5:+ k1 ^= ((long) data[index + 4] & 0xff) << 32;+ case 4:+ k1 ^= ((long) data[index + 3] & 0xff) << 24;+ case 3:+ k1 ^= ((long) data[index + 2] & 0xff) << 16;+ case 2:+ k1 ^= ((long) data[index + 1] & 0xff) << 8;+ case 1:+ k1 ^= data[index] & 0xff;+ k1 *= C1;+ k1 = Long.rotateLeft(k1, R1);+ k1 *= C2;+ h1 ^= k1;+ }++ // finalization+ h1 ^= length;+ h2 ^= length;++ h1 += h2;+ h2 += h1;++ h1 = fmix64(h1);+ h2 = fmix64(h2);++ h1 += h2;+ h2 += h1;++ return new long[] {h1, h2};+ }++ /**+ * Performs the final avalanche mix step of the 64-bit hash function.+ *+ * @param hash The current hash+ * @return The final hash+ */+ private static long fmix64(long hash) {+ hash ^= (hash >>> 33);+ hash *= 0xff51afd7ed558ccdL;+ hash ^= (hash >>> 33);+ hash *= 0xc4ceb9fe1a85ec53L;+ hash ^= (hash >>> 33);+ return hash;+ }++ /**+ * Generates 128-bit hash from the byte array with the given offset, length and seed.+ *+ * <p>The code is adopted from Apache Commons (<a+ * href="https://commons.apache.org/proper/commons-codec/jacoco/org.apache.commons.codec.digest/MurmurHash3.java.html">link</a>)+ *+ * @param data The input data+ * @return The 128-bit hash (2 longs)+ */+ public static long[] murmurhash3_x64_128(BytesRef data) {+ return murmurhash3_x64_128(data.bytes, data.offset, data.length, 104729);+ }+// Holds 128 bit unsigned value:private static BigInteger nextId;private static final BigInteger mask128;