Apache Lucene open-source search software
Create vectorized versions of ScalarQuantizer.quantize and recalculateCorrectiveOffset (#14304)
Simon Cooper1e8a1468768b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex ee3304272cc..495af0794e7 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -35,6 +35,7 @@ Optimizations---------------------* GITHUB#14011: Reduce allocation rate in HNSW concurrent merge. (Viliam Durina)* GITHUB#14022: Optimize DFS marking of connected components in HNSW by reducing stack depth, improving performance and reducing allocations. (Viswanath Kuchibhotla)+* GITHUB#14304: Add SIMD optimizations for scalar quantized queries and indexing. (Simon Cooper)Bug Fixes---------------------diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.javaindex d118b58f321..1d97a97660d 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java@@ -234,4 +234,79 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {}return ret;}++ @Override+ public float minMaxScalarQuantize(+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {+ return new ScalarQuantizer(alpha, scale, minQuantile, maxQuantile).quantize(vector, dest, 0);+ }++ @Override+ public float recalculateScalarQuantizationOffset(+ byte[] vector,+ float oldAlpha,+ float oldMinQuantile,+ float scale,+ float alpha,+ float minQuantile,+ float maxQuantile) {+ return new ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)+ .recalculateOffset(vector, 0, oldAlpha, oldMinQuantile);+ }++ static class ScalarQuantizer {+ private final float alpha;+ private final float scale;+ private final float minQuantile, maxQuantile;++ ScalarQuantizer(float alpha, float scale, float minQuantile, float maxQuantile) {+ this.alpha = alpha;+ this.scale = scale;+ this.minQuantile = minQuantile;+ this.maxQuantile = maxQuantile;+ }++ float quantize(float[] vector, byte[] dest, int start) {+ assert vector.length == dest.length;+ float correction = 0;+ for (int i = start; i < vector.length; i++) {+ correction += quantizeFloat(vector[i], dest, i);+ }+ return correction;+ }++ float recalculateOffset(byte[] vector, int start, float oldAlpha, float oldMinQuantile) {+ float correction = 0;+ for (int i = start; i < vector.length; i++) {+ // undo the old quantization+ float v = (oldAlpha * vector[i]) + oldMinQuantile;+ correction += quantizeFloat(v, null, 0);+ }+ return correction;+ }++ private float quantizeFloat(float v, byte[] dest, int destIndex) {+ assert dest == null || destIndex < dest.length;+ // Make sure the value is within the quantile range, cutting off the tails+ // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -+ // minQuantile)+ float dx = v - minQuantile;+ float dxc = Math.max(minQuantile, Math.min(maxQuantile, v)) - minQuantile;+ // Scale the value to the range [0, 127], this is our quantized value+ // scale = 127/(maxQuantile - minQuantile)+ int roundedDxs = Math.round(scale * dxc);+ // We multiply by `alpha` here to get the quantized value back into the original range+ // to aid in calculating the corrective offset+ float dxq = roundedDxs * alpha;+ if (dest != null) {+ dest[destIndex] = (byte) roundedDxs;+ }+ // Calculate the corrective offset that needs to be applied to the score+ // in addition to the `byte * minQuantile * alpha` term in the equation+ // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value+ // will be rounded to the nearest whole number and lose some accuracy+ // Additionally, we account for the global correction of `minQuantile^2` in the equation+ return minQuantile * (v - minQuantile / 2.0F) + (dx - dxq) * dxq;+ }+ }}diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.javaindex 82947d00343..0edeefbe353 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java@@ -65,4 +65,39 @@ public interface VectorUtilSupport {* @return the dot product*/long int4BitDotProduct(byte[] int4Quantized, byte[] binaryQuantized);++ /**+ * Quantizes {@code vector}, putting the result into {@code dest}.+ *+ * @param vector the vector to quantize+ * @param dest the destination vector+ * @param scale the scaling factor+ * @param alpha the alpha value+ * @param minQuantile the lower quantile of the distribution+ * @param maxQuantile the upper quantile of the distribution+ * @return the corrective offset that needs to be applied to the score+ */+ float minMaxScalarQuantize(+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile);++ /**+ * Recalculates the offset for {@code vector}.+ *+ * @param vector the vector to quantize+ * @param oldAlpha the previous alpha value+ * @param oldMinQuantile the previous lower quantile+ * @param scale the scaling factor+ * @param alpha the alpha value+ * @param minQuantile the lower quantile of the distribution+ * @param maxQuantile the upper quantile of the distribution+ * @return the new corrective offset+ */+ float recalculateScalarQuantizationOffset(+ byte[] vector,+ float oldAlpha,+ float oldMinQuantile,+ float scale,+ float alpha,+ float minQuantile,+ float maxQuantile);}diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.javaindex 75998c2f7fb..a64c5165ddf 100644--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java@@ -334,4 +334,46 @@ public final class VectorUtil {assert IntStream.range(0, to - 1).noneMatch(i -> buffer[i] > buffer[i + 1]);return IMPL.findNextGEQ(buffer, target, from, to);}++ /**+ * Scalar quantizes {@code vector}, putting the result into {@code dest}.+ *+ * @param vector the vector to quantize+ * @param dest the destination vector+ * @param scale the scaling factor+ * @param alpha the alpha value+ * @param minQuantile the lower quantile of the distribution+ * @param maxQuantile the upper quantile of the distribution+ * @return the corrective offset that needs to be applied to the score+ */+ public static float minMaxScalarQuantize(+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {+ if (vector.length != dest.length)+ throw new IllegalArgumentException("source and destination arrays should be the same size");+ return IMPL.minMaxScalarQuantize(vector, dest, scale, alpha, minQuantile, maxQuantile);+ }++ /**+ * Recalculates the offset for {@code vector}.+ *+ * @param vector the vector to quantize+ * @param oldAlpha the previous alpha value+ * @param oldMinQuantile the previous lower quantile+ * @param scale the scaling factor+ * @param alpha the alpha value+ * @param minQuantile the lower quantile of the distribution+ * @param maxQuantile the upper quantile of the distribution+ * @return the new corrective offset+ */+ public static float recalculateOffset(+ byte[] vector,+ float oldAlpha,+ float oldMinQuantile,+ float scale,+ float alpha,+ float minQuantile,+ float maxQuantile) {+ return IMPL.recalculateScalarQuantizationOffset(+ vector, oldAlpha, oldMinQuantile, scale, alpha, minQuantile, maxQuantile);+ }}diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.javaindex 34fb380a183..d6e2c0986a2 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java@@ -122,40 +122,15 @@ public class ScalarQuantizer {public float quantize(float[] src, byte[] dest, VectorSimilarityFunction similarityFunction) {assert src.length == dest.length;assert similarityFunction != VectorSimilarityFunction.COSINE || VectorUtil.isUnitVector(src);- float correction = 0;- for (int i = 0; i < src.length; i++) {- correction += quantizeFloat(src[i], dest, i);- }++ float correction =+ VectorUtil.minMaxScalarQuantize(src, dest, scale, alpha, minQuantile, maxQuantile);if (similarityFunction.equals(VectorSimilarityFunction.EUCLIDEAN)) {return 0;}return correction;}- private float quantizeFloat(float v, byte[] dest, int destIndex) {- assert dest == null || destIndex < dest.length;- // Make sure the value is within the quantile range, cutting off the tails- // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -- // minQuantile)- float dx = v - minQuantile;- float dxc = Math.max(minQuantile, Math.min(maxQuantile, v)) - minQuantile;- // Scale the value to the range [0, 127], this is our quantized value- // scale = 127/(maxQuantile - minQuantile)- float dxs = scale * dxc;- // We multiply by `alpha` here to get the quantized value back into the original range- // to aid in calculating the corrective offset- float dxq = Math.round(dxs) * alpha;- if (dest != null) {- dest[destIndex] = (byte) Math.round(dxs);- }- // Calculate the corrective offset that needs to be applied to the score- // in addition to the `byte * minQuantile * alpha` term in the equation- // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value- // will be rounded to the nearest whole number and lose some accuracy- // Additionally, we account for the global correction of `minQuantile^2` in the equation- return minQuantile * (v - minQuantile / 2.0F) + (dx - dxq) * dxq;- }-/*** Recalculate the old score corrective value given new current quantiles*@@ -171,13 +146,14 @@ public class ScalarQuantizer {if (similarityFunction.equals(VectorSimilarityFunction.EUCLIDEAN)) {return 0f;}- float correctiveOffset = 0f;- for (int i = 0; i < quantizedVector.length; i++) {- // dequantize the old value in order to recalculate the corrective offset- float v = (oldQuantizer.alpha * quantizedVector[i]) + oldQuantizer.minQuantile;- correctiveOffset += quantizeFloat(v, null, 0);- }- return correctiveOffset;+ return VectorUtil.recalculateOffset(+ quantizedVector,+ oldQuantizer.alpha,+ oldQuantizer.minQuantile,+ scale,+ alpha,+ minQuantile,+ maxQuantile);}/**diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex b886611bba1..9aa4bc09e34 100644--- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -907,4 +907,98 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {}return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3);}++ @Override+ public float minMaxScalarQuantize(+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {+ assert vector.length == dest.length;+ float correction = 0;+ int i = 0;+ // only vectorize if we have a viable BYTE_SPECIES we can use for output+ if (VECTOR_BITSIZE >= 256) {+ FloatVector sum = FloatVector.zero(FLOAT_SPECIES);++ for (; i < FLOAT_SPECIES.loopBound(vector.length); i += FLOAT_SPECIES.length()) {+ FloatVector v = FloatVector.fromArray(FLOAT_SPECIES, vector, i);++ // Make sure the value is within the quantile range, cutting off the tails+ // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -+ // minQuantile)+ FloatVector dxc = v.min(maxQuantile).max(minQuantile).sub(minQuantile);+ // Scale the value to the range [0, 127], this is our quantized value+ // scale = 127/(maxQuantile - minQuantile)+ // Math.round rounds to positive infinity, so do the same by +0.5 then truncating to int+ Vector<Integer> roundedDxs =+ fma(dxc, dxc.broadcast(scale), dxc.broadcast(0.5f)).convert(VectorOperators.F2I, 0);+ // output this to the array+ ((ByteVector) roundedDxs.castShape(BYTE_SPECIES, 0)).intoArray(dest, i);+ // We multiply by `alpha` here to get the quantized value back into the original range+ // to aid in calculating the corrective offset+ FloatVector dxq = ((FloatVector) roundedDxs.castShape(FLOAT_SPECIES, 0)).mul(alpha);+ // Calculate the corrective offset that needs to be applied to the score+ // in addition to the `byte * minQuantile * alpha` term in the equation+ // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value+ // will be rounded to the nearest whole number and lose some accuracy+ // Additionally, we account for the global correction of `minQuantile^2` in the equation+ sum =+ fma(+ v.sub(minQuantile / 2f),+ v.broadcast(minQuantile),+ fma(v.sub(minQuantile).sub(dxq), dxq, sum));+ }++ correction = sum.reduceLanes(VectorOperators.ADD);+ }++ // complete the tail normally+ correction +=+ new DefaultVectorUtilSupport.ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)+ .quantize(vector, dest, i);++ return correction;+ }++ @Override+ public float recalculateScalarQuantizationOffset(+ byte[] vector,+ float oldAlpha,+ float oldMinQuantile,+ float scale,+ float alpha,+ float minQuantile,+ float maxQuantile) {+ float correction = 0;+ int i = 0;+ // only vectorize if we have a viable BYTE_SPECIES that we can use+ if (VECTOR_BITSIZE >= 256) {+ FloatVector sum = FloatVector.zero(FLOAT_SPECIES);++ for (; i < BYTE_SPECIES.loopBound(vector.length); i += BYTE_SPECIES.length()) {+ FloatVector fv =+ (FloatVector) ByteVector.fromArray(BYTE_SPECIES, vector, i).castShape(FLOAT_SPECIES, 0);+ // undo the old quantization+ FloatVector v = fma(fv, fv.broadcast(oldAlpha), fv.broadcast(oldMinQuantile));++ // same operations as in quantize above+ FloatVector dxc = v.min(maxQuantile).max(minQuantile).sub(minQuantile);+ Vector<Integer> roundedDxs =+ fma(dxc, dxc.broadcast(scale), dxc.broadcast(0.5f)).convert(VectorOperators.F2I, 0);+ FloatVector dxq = ((FloatVector) roundedDxs.castShape(FLOAT_SPECIES, 0)).mul(alpha);+ sum =+ fma(+ v.sub(minQuantile / 2f),+ v.broadcast(minQuantile),+ fma(v.sub(minQuantile).sub(dxq), dxq, sum));+ }++ correction = sum.reduceLanes(VectorOperators.ADD);+ }++ // complete the tail normally+ correction +=+ new DefaultVectorUtilSupport.ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)+ .recalculateOffset(vector, i, oldAlpha, oldMinQuantile);++ return correction;+ }}diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.javaindex 6443de752bf..ee62e839c06 100644--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java@@ -16,8 +16,13 @@*/package org.apache.lucene.internal.vectorization;+import static org.hamcrest.Matchers.closeTo;+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.util.ArrayList;import java.util.Arrays;+import java.util.List;+import java.util.Random;import java.util.function.ToDoubleFunction;import java.util.function.ToIntFunction;import java.util.function.ToLongFunction;@@ -25,17 +30,18 @@ import java.util.stream.IntStream;public class TestVectorUtilSupport extends BaseVectorizationTestCase {- private static final double DELTA = 1e-3;-private static final int[] VECTOR_SIZES = {1, 4, 6, 8, 13, 16, 25, 32, 64, 100, 128, 207, 256, 300, 512, 702, 1024, 1536, 2046, 2048, 4096,4098};private final int size;+ private final double delta;public TestVectorUtilSupport(int size) {this.size = size;+ // scale the delta with the size+ this.delta = 1e-6 * size;}@ParametersFactory@@ -164,11 +170,49 @@ public class TestVectorUtilSupport extends BaseVectorizationTestCase {return packed;}+ public void testMinMaxScalarQuantize() {+ Random r = random();+ float min = r.nextFloat(-1, 1);+ float max = r.nextFloat(min, 1);+ float divisor = (float) ((1 << 7) - 1); // 7 bits quantization here++ float scale = divisor / (max - min);+ float alpha = (max - min) / divisor;++ float[] vector = new float[size];+ for (int i = 0; i < vector.length; i++) {+ vector[i] = (r.nextFloat() * (max - min)) + min;+ }++ List<byte[]> outputs = new ArrayList<>();+ assertFloatReturningProviders(+ p -> {+ byte[] output = new byte[size];+ outputs.add(output);+ return p.minMaxScalarQuantize(vector, output, scale, alpha, min, max);+ });++ // check the outputs are identical+ for (int o = 1; o < outputs.size(); o++) {+ assertArrayEquals(outputs.getFirst(), outputs.get(o));+ }++ // check recalculation too+ float newMax = max * 2;+ float newMin = min / 2;+ float newScale = divisor / (newMax - newMin);+ float newAlpha = (newMax - newMin) / divisor;++ assertFloatReturningProviders(+ p ->+ p.recalculateScalarQuantizationOffset(+ outputs.getFirst(), alpha, min, newScale, newAlpha, newMin, newMax));+ }+private void assertFloatReturningProviders(ToDoubleFunction<VectorUtilSupport> func) {- assertEquals(- func.applyAsDouble(LUCENE_PROVIDER.getVectorUtilSupport()),+ assertThat(func.applyAsDouble(PANAMA_PROVIDER.getVectorUtilSupport()),- DELTA);+ closeTo(func.applyAsDouble(LUCENE_PROVIDER.getVectorUtilSupport()), delta));}private void assertIntReturningProviders(ToIntFunction<VectorUtilSupport> func) {