Apache Lucene open-source search software
Add support for uint8 distance comparison (#15148)
Trevor McCullochef9128ffe058
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 7fdeedfec43..28aeee6b0e4 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -123,7 +123,7 @@ New FeaturesImprovements----------------------(No changes)+# GITHUB#15148: Add support uint8 distance and allow 8 bit scalar quantization (Trevor McCulloch)Optimizations---------------------diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorUtilBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorUtilBenchmark.javaindex f3aa61ba8be..a8eb1b945ce 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorUtilBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorUtilBenchmark.java@@ -119,6 +119,17 @@ public class VectorUtilBenchmark {return VectorUtil.dotProduct(bytesA, bytesB);}+ @Benchmark+ public int binaryDotProductUint8Scalar() {+ return VectorUtil.uint8DotProduct(bytesA, bytesB);+ }++ @Benchmark+ @Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})+ public int binaryDotProductUint8Vector() {+ return VectorUtil.uint8DotProduct(bytesA, bytesB);+ }+@Benchmarkpublic int binarySquareScalar() {return VectorUtil.squareDistance(bytesA, bytesB);@@ -130,6 +141,17 @@ public class VectorUtilBenchmark {return VectorUtil.squareDistance(bytesA, bytesB);}+ @Benchmark+ public int binarySquareUint8Scalar() {+ return VectorUtil.uint8SquareDistance(bytesA, bytesB);+ }++ @Benchmark+ @Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})+ public int binarySquareUint8Vector() {+ return VectorUtil.uint8SquareDistance(bytesA, bytesB);+ }+@Benchmarkpublic int binaryHalfByteScalar() {return VectorUtil.int4DotProduct(halfBytesA, halfBytesB);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.javaindex 9153e0da38e..117521ddcc2 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java@@ -156,7 +156,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {@Overridepublic float score(int node) throws IOException {byte[] nodeVector = values.vectorValue(node);- int squareDistance = VectorUtil.squareDistance(nodeVector, targetBytes);+ int squareDistance = VectorUtil.uint8SquareDistance(nodeVector, targetBytes);float adjustedDistance = squareDistance * constMultiplier;return 1 / (1f + adjustedDistance);}@@ -194,8 +194,9 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {public float score(int vectorOrdinal) throws IOException {byte[] storedVector = values.vectorValue(vectorOrdinal);float vectorOffset = values.getScoreCorrectionConstant(vectorOrdinal);- int dotProduct = VectorUtil.dotProduct(storedVector, targetBytes);- // For the current implementation of scalar quantization, all dotproducts should be >= 0;+ int dotProduct = VectorUtil.uint8DotProduct(storedVector, targetBytes);+ // For the current implementation of scalar quantization, all dotproducts should+ // be >= 0;assert dotProduct >= 0;float adjustedDistance = dotProduct * constMultiplier + offsetCorrection + vectorOffset;return scoreAdjustmentFunction.apply(adjustedDistance);@@ -208,9 +209,10 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {}}- // TODO consider splitting this into two classes. right now the "query" vector is always+ // TODO consider splitting this into two classes. right now the "query" vector+ // is always// decompressed- // it could stay compressed if we had a compressed version of the target vector+ // it could stay compressed if we had a compressed version of the target vectorprivate static class CompressedInt4DotProductextends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {private final float constMultiplier;@@ -237,13 +239,15 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {@Overridepublic float score(int vectorOrdinal) throws IOException {- // get compressed vector, in Lucene99, vector values are stored and have a single value for+ // get compressed vector, in Lucene99, vector values are stored and have a+ // single value for// offset correctionvalues.getSlice().seek((long) vectorOrdinal * (values.getVectorByteLength() + Float.BYTES));values.getSlice().readBytes(compressedVector, 0, compressedVector.length);float vectorOffset = values.getScoreCorrectionConstant(vectorOrdinal);int dotProduct = VectorUtil.int4DotProductPacked(targetBytes, compressedVector);- // For the current implementation of scalar quantization, all dotproducts should be >= 0;+ // For the current implementation of scalar quantization, all dotproducts should+ // be >= 0;assert dotProduct >= 0;float adjustedDistance = dotProduct * constMultiplier + offsetCorrection + vectorOffset;return scoreAdjustmentFunction.apply(adjustedDistance);@@ -283,7 +287,8 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {byte[] storedVector = values.vectorValue(vectorOrdinal);float vectorOffset = values.getScoreCorrectionConstant(vectorOrdinal);int dotProduct = VectorUtil.int4DotProduct(storedVector, targetBytes);- // For the current implementation of scalar quantization, all dotproducts should be >= 0;+ // For the current implementation of scalar quantization, all dotproducts should+ // be >= 0;assert dotProduct >= 0;float adjustedDistance = dotProduct * constMultiplier + offsetCorrection + vectorOffset;return scoreAdjustmentFunction.apply(adjustedDistance);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.javaindex 0b3c6d19af8..0f339ecbe0a 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java@@ -34,10 +34,8 @@ import org.apache.lucene.index.SegmentWriteState;public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {// The bits that are allowed for scalar quantization- // We only allow signed byte (7), and half-byte (4)- // NOTE: we used to allow 8 bits as well, but it was broken so we removed it- // (https://github.com/apache/lucene/issues/13519)- private static final int ALLOWED_BITS = (1 << 7) | (1 << 4);+ // We only allow unsigned byte (8), signed byte (7), and half-byte (4)+ private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4);public static final String QUANTIZED_VECTOR_COMPONENT = "QVEC";public static final String NAME = "Lucene99ScalarQuantizedVectorsFormat";diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.javaindex b991a81ce07..89c813a4b93 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java@@ -154,6 +154,15 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {return total;}+ @Override+ public int uint8DotProduct(byte[] a, byte[] b) {+ int total = 0;+ for (int i = 0; i < a.length; i++) {+ total += Byte.toUnsignedInt(a[i]) * Byte.toUnsignedInt(b[i]);+ }+ return total;+ }+@Overridepublic int int4DotProduct(byte[] a, boolean apacked, byte[] b, boolean bpacked) {assert (apacked && bpacked) == false;@@ -201,6 +210,17 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {return squareSum;}+ @Override+ public int uint8SquareDistance(byte[] a, byte[] b) {+ // Note: this will not overflow if dim < 2^16, since max(ubyte * ubyte) = 2^16.+ int squareSum = 0;+ for (int i = 0; i < a.length; i++) {+ int diff = Byte.toUnsignedInt(a[i]) - Byte.toUnsignedInt(b[i]);+ squareSum += diff * diff;+ }+ return squareSum;+ }+@Overridepublic int findNextGEQ(int[] buffer, int target, int from, int to) {for (int i = from; i < to; ++i) {@@ -281,7 +301,7 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {float correction = 0;for (int i = start; i < vector.length; i++) {// undo the old quantization- float v = (oldAlpha * vector[i]) + oldMinQuantile;+ float v = (oldAlpha * Byte.toUnsignedInt(vector[i])) + oldMinQuantile;correction += quantizeFloat(v, null, 0);}return correction;diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.javaindex 82db11aa8d4..7190f983b4c 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java@@ -36,6 +36,9 @@ public interface VectorUtilSupport {/** Returns the dot product computed over signed bytes. */int dotProduct(byte[] a, byte[] b);+ /** Returns the dot product computed as though the bytes were unsigned. */+ int uint8DotProduct(byte[] a, byte[] b);+/** Returns the dot product over the computed bytes, assuming the values are int4 encoded. */int int4DotProduct(byte[] a, boolean apacked, byte[] b, boolean bpacked);@@ -45,6 +48,9 @@ public interface VectorUtilSupport {/** Returns the sum of squared differences of the two byte vectors. */int squareDistance(byte[] a, byte[] b);+ /** Returns the sum of squared differences of the two unsigned byte vectors. */+ int uint8SquareDistance(byte[] a, byte[] b);+/*** Given an array {@code buffer} that is sorted between indexes {@code 0} inclusive and {@code to}* exclusive, find the first array index whose value is greater than or equal to {@code target}.diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.javaindex a09afed147b..38b9cf6d67a 100644--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java@@ -113,6 +113,14 @@ public final class VectorUtil {return IMPL.squareDistance(a, b);}+ /** Returns the sum of squared differences of the two vectors where each byte is unsigned */+ public static int uint8SquareDistance(byte[] a, byte[] b) {+ if (a.length != b.length) {+ throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length);+ }+ return IMPL.uint8SquareDistance(a, b);+ }+/*** Modifies the argument to be unit length, dividing by its l2-norm. IllegalArgumentException is* thrown for zero vectors.@@ -167,6 +175,20 @@ public final class VectorUtil {return IMPL.dotProduct(a, b);}+ /**+ * Dot product over bytes assuming that the values are actually unsigned.+ *+ * @param a uint8 byte vector+ * @param b another uint8 byte vector of the same dimension+ * @return the value of the dot product of the two vectors+ */+ public static int uint8DotProduct(byte[] a, byte[] b) {+ if (a.length != b.length) {+ throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length);+ }+ return IMPL.uint8DotProduct(a, b);+ }+public static int int4DotProduct(byte[] a, byte[] b) {if (a.length != b.length) {throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length);diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizedVectorSimilarity.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizedVectorSimilarity.javaindex 1c22d191f51..9ce8ed42e5f 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizedVectorSimilarity.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizedVectorSimilarity.java@@ -42,10 +42,12 @@ public interface ScalarQuantizedVectorSimilarity {case EUCLIDEAN -> new Euclidean(constMultiplier);case COSINE, DOT_PRODUCT ->new DotProduct(- constMultiplier, bits <= 4 ? VectorUtil::int4DotProduct : VectorUtil::dotProduct);+ constMultiplier,+ bits <= 4 ? VectorUtil::int4DotProduct : VectorUtil::uint8DotProduct);case MAXIMUM_INNER_PRODUCT ->new MaximumInnerProduct(- constMultiplier, bits <= 4 ? VectorUtil::int4DotProduct : VectorUtil::dotProduct);+ constMultiplier,+ bits <= 4 ? VectorUtil::int4DotProduct : VectorUtil::uint8DotProduct);};}@@ -62,7 +64,7 @@ public interface ScalarQuantizedVectorSimilarity {@Overridepublic float score(byte[] queryVector, float queryVectorOffset, byte[] storedVector, float vectorOffset) {- int squareDistance = VectorUtil.squareDistance(storedVector, queryVector);+ int squareDistance = VectorUtil.uint8SquareDistance(storedVector, queryVector);float adjustedDistance = squareDistance * constMultiplier;return 1 / (1f + adjustedDistance);}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.javaindex 04fa2c98b9f..cacc86e2ef1 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java@@ -165,7 +165,7 @@ public class ScalarQuantizer {public void deQuantize(byte[] src, float[] dest) {assert src.length == dest.length;for (int i = 0; i < src.length; i++) {- dest[i] = (alpha * src[i]) + minQuantile;+ dest[i] = (alpha * Byte.toUnsignedInt(src[i])) + minQuantile;}}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex e7a989aa3c7..a77c4846ca2 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -23,7 +23,9 @@ import static jdk.incubator.vector.VectorOperators.B2I;import static jdk.incubator.vector.VectorOperators.B2S;import static jdk.incubator.vector.VectorOperators.LSHR;import static jdk.incubator.vector.VectorOperators.S2I;+import static jdk.incubator.vector.VectorOperators.ZERO_EXTEND_B2I;import static jdk.incubator.vector.VectorOperators.ZERO_EXTEND_B2S;+import static jdk.incubator.vector.VectorOperators.ZERO_EXTEND_S2I;import static org.apache.lucene.util.VectorUtil.EPSILON;import java.lang.foreign.MemorySegment;@@ -361,6 +363,11 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return dotProductBody(new ArrayLoader(a), new ArrayLoader(b));}+ @Override+ public int uint8DotProduct(byte[] a, byte[] b) {+ return dotProductBody(new ArrayLoader(a), new ArrayLoader(b), false);+ }+public static int dotProduct(byte[] a, MemorySegment b) {return dotProductBody(new ArrayLoader(a), new MemorySegmentLoader(b));}@@ -370,6 +377,10 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}private static int dotProductBody(ByteVectorLoader a, ByteVectorLoader b) {+ return dotProductBody(a, b, true);+ }++ private static int dotProductBody(ByteVectorLoader a, ByteVectorLoader b, boolean signed) {assert a.length() == b.length();int i = 0;int res = 0;@@ -379,38 +390,47 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {// compute vectorized dot product consistent with VPDPBUSD instructionif (VECTOR_BITSIZE >= 512) {i += BYTE_SPECIES.loopBound(a.length());- res += dotProductBody512(a, b, i);+ res += dotProductBody512(a, b, i, signed);} else if (VECTOR_BITSIZE == 256) {i += BYTE_SPECIES.loopBound(a.length());- res += dotProductBody256(a, b, i);+ res += dotProductBody256(a, b, i, signed);} else {// tricky: we don't have SPECIES_32, so we workaround with "overlapping read"i += ByteVector.SPECIES_64.loopBound(a.length() - ByteVector.SPECIES_64.length());- res += dotProductBody128(a, b, i);+ res += dotProductBody128(a, b, i, signed);}}// scalar tail- for (; i < a.length(); i++) {- res += a.tail(i) * b.tail(i);+ if (signed) {+ for (; i < a.length(); i++) {+ res += a.tail(i) * b.tail(i);+ }+ } else {+ for (; i < a.length(); i++) {+ res += Byte.toUnsignedInt(a.tail(i)) * Byte.toUnsignedInt(b.tail(i));+ }}return res;}/** vectorized dot product body (512 bit vectors) */- private static int dotProductBody512(ByteVectorLoader a, ByteVectorLoader b, int limit) {+ private static int dotProductBody512(+ ByteVectorLoader a, ByteVectorLoader b, int limit, boolean signed) {IntVector acc = IntVector.zero(INT_SPECIES);+ var conversion_short = signed ? B2S : ZERO_EXTEND_B2S;+ var conversion_int = signed ? S2I : ZERO_EXTEND_S2I;for (int i = 0; i < limit; i += BYTE_SPECIES.length()) {ByteVector va8 = a.load(BYTE_SPECIES, i);ByteVector vb8 = b.load(BYTE_SPECIES, i);// 16-bit multiply: avoid AVX-512 heavy multiply on zmm- Vector<Short> va16 = va8.convertShape(B2S, SHORT_SPECIES, 0);- Vector<Short> vb16 = vb8.convertShape(B2S, SHORT_SPECIES, 0);+ Vector<Short> va16 = va8.convertShape(conversion_short, SHORT_SPECIES, 0);+ Vector<Short> vb16 = vb8.convertShape(conversion_short, SHORT_SPECIES, 0);Vector<Short> prod16 = va16.mul(vb16);// 32-bit add- Vector<Integer> prod32 = prod16.convertShape(S2I, INT_SPECIES, 0);+ Vector<Integer> prod32 = prod16.convertShape(conversion_int, INT_SPECIES, 0);acc = acc.add(prod32);}// reduce@@ -418,15 +438,17 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}/** vectorized dot product body (256 bit vectors) */- private static int dotProductBody256(ByteVectorLoader a, ByteVectorLoader b, int limit) {+ private static int dotProductBody256(+ ByteVectorLoader a, ByteVectorLoader b, int limit, boolean signed) {IntVector acc = IntVector.zero(IntVector.SPECIES_256);+ var conversion = signed ? B2I : ZERO_EXTEND_B2I;for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length()) {ByteVector va8 = a.load(ByteVector.SPECIES_64, i);ByteVector vb8 = b.load(ByteVector.SPECIES_64, i);// 32-bit multiply and add into accumulator- Vector<Integer> va32 = va8.convertShape(B2I, IntVector.SPECIES_256, 0);- Vector<Integer> vb32 = vb8.convertShape(B2I, IntVector.SPECIES_256, 0);+ Vector<Integer> va32 = va8.convertShape(conversion, IntVector.SPECIES_256, 0);+ Vector<Integer> vb32 = vb8.convertShape(conversion, IntVector.SPECIES_256, 0);acc = acc.add(va32.mul(vb32));}// reduce@@ -434,8 +456,11 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}/** vectorized dot product body (128 bit vectors) */- private static int dotProductBody128(ByteVectorLoader a, ByteVectorLoader b, int limit) {+ private static int dotProductBody128(+ ByteVectorLoader a, ByteVectorLoader b, int limit, boolean signed) {IntVector acc = IntVector.zero(IntVector.SPECIES_128);+ var conversion_short = signed ? B2S : ZERO_EXTEND_B2S;+ var conversion_int = signed ? S2I : ZERO_EXTEND_S2I;// 4 bytes at a time (re-loading half the vector each time!)for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length() >> 1) {// load 8 bytes@@ -443,12 +468,12 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {ByteVector vb8 = b.load(ByteVector.SPECIES_64, i);// process first "half" only: 16-bit multiply- Vector<Short> va16 = va8.convert(B2S, 0);- Vector<Short> vb16 = vb8.convert(B2S, 0);+ Vector<Short> va16 = va8.convert(conversion_short, 0);+ Vector<Short> vb16 = vb8.convert(conversion_short, 0);Vector<Short> prod16 = va16.mul(vb16);// 32-bit add- acc = acc.add(prod16.convertShape(S2I, IntVector.SPECIES_128, 0));+ acc = acc.add(prod16.convertShape(conversion_int, IntVector.SPECIES_128, 0));}// reducereturn acc.reduceLanes(ADD);@@ -766,6 +791,11 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return squareDistanceBody(new ArrayLoader(a), new ArrayLoader(b));}+ @Override+ public int uint8SquareDistance(byte[] a, byte[] b) {+ return squareDistanceBody(new ArrayLoader(a), new ArrayLoader(b), false);+ }+public static int squareDistance(MemorySegment a, MemorySegment b) {return squareDistanceBody(new MemorySegmentLoader(a), new MemorySegmentLoader(b));}@@ -775,6 +805,10 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}private static int squareDistanceBody(ByteVectorLoader a, ByteVectorLoader b) {+ return squareDistanceBody(a, b, true);+ }++ private static int squareDistanceBody(ByteVectorLoader a, ByteVectorLoader b, boolean signed) {assert a.length() == b.length();int i = 0;int res = 0;@@ -783,32 +817,41 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {if (a.length() >= 16) {if (VECTOR_BITSIZE >= 256) {i += BYTE_SPECIES.loopBound(a.length());- res += squareDistanceBody256(a, b, i);+ res += squareDistanceBody256(a, b, i, signed);} else {i += ByteVector.SPECIES_64.loopBound(a.length());- res += squareDistanceBody128(a, b, i);+ res += squareDistanceBody128(a, b, i, signed);}}// scalar tail- for (; i < a.length(); i++) {- int diff = a.tail(i) - b.tail(i);- res += diff * diff;+ if (signed) {+ for (; i < a.length(); i++) {+ int diff = a.tail(i) - b.tail(i);+ res += diff * diff;+ }+ } else {+ for (; i < a.length(); i++) {+ int diff = Byte.toUnsignedInt(a.tail(i)) - Byte.toUnsignedInt(b.tail(i));+ res += diff * diff;+ }}return res;}/** vectorized square distance body (256+ bit vectors) */- private static int squareDistanceBody256(ByteVectorLoader a, ByteVectorLoader b, int limit) {+ private static int squareDistanceBody256(+ ByteVectorLoader a, ByteVectorLoader b, int limit, boolean signed) {IntVector acc = IntVector.zero(INT_SPECIES);+ var conversion = signed ? B2I : ZERO_EXTEND_B2I;for (int i = 0; i < limit; i += BYTE_SPECIES.length()) {ByteVector va8 = a.load(BYTE_SPECIES, i);ByteVector vb8 = b.load(BYTE_SPECIES, i);// 32-bit sub, multiply, and add into accumulators// TODO: uses AVX-512 heavy multiply on zmm, should we just use 256-bit vectors on AVX-512?- Vector<Integer> va32 = va8.convertShape(B2I, INT_SPECIES, 0);- Vector<Integer> vb32 = vb8.convertShape(B2I, INT_SPECIES, 0);+ Vector<Integer> va32 = va8.convertShape(conversion, INT_SPECIES, 0);+ Vector<Integer> vb32 = vb8.convertShape(conversion, INT_SPECIES, 0);Vector<Integer> diff32 = va32.sub(vb32);acc = acc.add(diff32.mul(diff32));}@@ -817,18 +860,20 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}/** vectorized square distance body (128 bit vectors) */- private static int squareDistanceBody128(ByteVectorLoader a, ByteVectorLoader b, int limit) {+ private static int squareDistanceBody128(+ ByteVectorLoader a, ByteVectorLoader b, int limit, boolean signed) {// 128-bit implementation, which must "split up" vectors due to widening conversions// it doesn't help to do the overlapping read trick, due to 32-bit multiply in the formulaIntVector acc1 = IntVector.zero(IntVector.SPECIES_128);IntVector acc2 = IntVector.zero(IntVector.SPECIES_128);+ var conversion_short = signed ? B2S : ZERO_EXTEND_B2S;for (int i = 0; i < limit; i += ByteVector.SPECIES_64.length()) {ByteVector va8 = a.load(ByteVector.SPECIES_64, i);ByteVector vb8 = b.load(ByteVector.SPECIES_64, i);// 16-bit sub- Vector<Short> va16 = va8.convertShape(B2S, ShortVector.SPECIES_128, 0);- Vector<Short> vb16 = vb8.convertShape(B2S, ShortVector.SPECIES_128, 0);+ Vector<Short> va16 = va8.convertShape(conversion_short, ShortVector.SPECIES_128, 0);+ Vector<Short> vb16 = vb8.convertShape(conversion_short, ShortVector.SPECIES_128, 0);Vector<Short> diff16 = va16.sub(vb16);// 32-bit multiply and add into accumulators@@ -1049,7 +1094,10 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {for (; i < BYTE_SPECIES.loopBound(vector.length); i += BYTE_SPECIES.length()) {FloatVector fv =- (FloatVector) ByteVector.fromArray(BYTE_SPECIES, vector, i).castShape(FLOAT_SPECIES, 0);+ (FloatVector)+ ByteVector.fromArray(BYTE_SPECIES, vector, i)+ .convertShape(ZERO_EXTEND_B2S, SHORT_SPECIES, 0)+ .castShape(FLOAT_SPECIES, 0);// undo the old quantizationFloatVector v = fma(fv, fv.broadcast(oldAlpha), fv.broadcast(oldMinQuantile));diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.javaindex c3be45cb644..7ec661b3659 100644--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java@@ -172,7 +172,7 @@ public class TestVectorUtilSupport extends BaseVectorizationTestCase {Random r = random();float min = r.nextFloat(-1, 1);float max = r.nextFloat(min, 1);- float divisor = (float) ((1 << 7) - 1); // 7 bits quantization here+ float divisor = (float) ((1 << 8) - 1); // 8 bit quantization herefloat scale = divisor / (max - min);float alpha = (max - min) / divisor;diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java b/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.javaindex e6f3380060d..1953d6dff75 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java@@ -179,6 +179,14 @@ public class TestVectorUtil extends LuceneTestCase {return l2;}+ private static float uint8L2(byte[] v) {+ float l2 = 0;+ for (int i = 0; i < v.length; i++) {+ l2 += Byte.toUnsignedInt(v[i]) * Byte.toUnsignedInt(v[i]);+ }+ return l2;+ }+private static float[] randomVector() {return randomVector(random().nextInt(100) + 1);}@@ -270,6 +278,33 @@ public class TestVectorUtil extends LuceneTestCase {assertEquals(4 * l2(v), VectorUtil.squareDistance(u, v), DELTA);}+ public void testBasicDotProductUint8() {+ byte[] a = new byte[] {1, 2, 3};+ byte[] b = new byte[] {-10, 0, 5};+ assertEquals(261, VectorUtil.uint8DotProduct(a, b), 0);++ byte[] min = new byte[] {-128, -128};+ byte[] max = new byte[] {127, 127};+ assertEquals(32512, VectorUtil.uint8DotProduct(min, max), DELTA);+ }++ public void testSelfDotProductUint8() {+ // the dot product of a vector with itself is equal to the sum of the squares of its components+ byte[] v = randomVectorBytes();+ assertEquals(uint8L2(v), VectorUtil.uint8DotProduct(v, v), DELTA);+ }++ public void testSelfSquareDistanceUint8() {+ // the l2 distance of a vector with itself is zero+ byte[] v = randomVectorBytes();+ assertEquals(0, VectorUtil.uint8SquareDistance(v, v), DELTA);+ }++ public void testBasicSquareDistanceUint8() {+ assertEquals(+ 64524, VectorUtil.uint8SquareDistance(new byte[] {1, 2, 3}, new byte[] {-1, 0, 5}), 0);+ }+public void testBasicCosineBytes() {assertEquals(0.11952f, VectorUtil.cosine(new byte[] {1, 2, 3}, new byte[] {-10, 0, 5}), DELTA);}diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizedVectorSimilarity.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizedVectorSimilarity.javaindex f2cc3ac35c0..b98f6bbc1fb 100644--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizedVectorSimilarity.java+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizedVectorSimilarity.java@@ -37,7 +37,7 @@ public class TestScalarQuantizedVectorSimilarity extends LuceneTestCase {if (random().nextBoolean()) {multiplier = -multiplier;}- for (byte bits : new byte[] {4, 7}) {+ for (byte bits : new byte[] {4, 7, 8}) {ScalarQuantizedVectorSimilarity quantizedSimilarity =ScalarQuantizedVectorSimilarity.fromVectorSimilarity(similarityFunction, multiplier, bits);diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizer.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizer.javaindex 40944662dee..323531e119a 100644--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizer.java+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestScalarQuantizer.java@@ -118,6 +118,29 @@ public class TestScalarQuantizer extends LuceneTestCase {assertTrue(minDimValue >= (byte) 0);}+ public void testQuantizeAndDeQuantize8Bit() throws IOException {+ int dims = 128;+ int numVecs = 100;+ VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.DOT_PRODUCT;++ float[][] floats = randomFloats(numVecs, dims);+ FloatVectorValues floatVectorValues = fromFloats(floats);+ ScalarQuantizer scalarQuantizer =+ ScalarQuantizer.fromVectors(floatVectorValues, 1, numVecs, (byte) 8);+ float[] dequantized = new float[dims];+ byte[] quantized = new byte[dims];+ byte[] requantized = new byte[dims];+ for (int i = 0; i < numVecs; i++) {+ scalarQuantizer.quantize(floats[i], quantized, similarityFunction);+ scalarQuantizer.deQuantize(quantized, dequantized);+ scalarQuantizer.quantize(dequantized, requantized, similarityFunction);+ for (int j = 0; j < dims; j++) {+ assertEquals(dequantized[j], floats[i][j], 0.02);+ assertEquals(quantized[j], requantized[j]);+ }+ }+ }+public void testQuantiles() {float[] percs = new float[1000];for (int i = 0; i < 1000; i++) {