Skip to content

Apache Lucene open-source search software

Create vectorized versions of ScalarQuantizer.quantize and recalculateCorrectiveOffset (#14304)

Simon Cooper1e8a1468768b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ee3304272cc..495af0794e7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -35,6 +35,7 @@ Optimizations
---------------------
* GITHUB#14011: Reduce allocation rate in HNSW concurrent merge. (Viliam Durina)
* GITHUB#14022: Optimize DFS marking of connected components in HNSW by reducing stack depth, improving performance and reducing allocations. (Viswanath Kuchibhotla)
+* GITHUB#14304: Add SIMD optimizations for scalar quantized queries and indexing. (Simon Cooper)
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java
index d118b58f321..1d97a97660d 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java
@@ -234,4 +234,79 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {
}
return ret;
}
+
+ @Override
+ public float minMaxScalarQuantize(
+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {
+ return new ScalarQuantizer(alpha, scale, minQuantile, maxQuantile).quantize(vector, dest, 0);
+ }
+
+ @Override
+ public float recalculateScalarQuantizationOffset(
+ byte[] vector,
+ float oldAlpha,
+ float oldMinQuantile,
+ float scale,
+ float alpha,
+ float minQuantile,
+ float maxQuantile) {
+ return new ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)
+ .recalculateOffset(vector, 0, oldAlpha, oldMinQuantile);
+ }
+
+ static class ScalarQuantizer {
+ private final float alpha;
+ private final float scale;
+ private final float minQuantile, maxQuantile;
+
+ ScalarQuantizer(float alpha, float scale, float minQuantile, float maxQuantile) {
+ this.alpha = alpha;
+ this.scale = scale;
+ this.minQuantile = minQuantile;
+ this.maxQuantile = maxQuantile;
+ }
+
+ float quantize(float[] vector, byte[] dest, int start) {
+ assert vector.length == dest.length;
+ float correction = 0;
+ for (int i = start; i < vector.length; i++) {
+ correction += quantizeFloat(vector[i], dest, i);
+ }
+ return correction;
+ }
+
+ float recalculateOffset(byte[] vector, int start, float oldAlpha, float oldMinQuantile) {
+ float correction = 0;
+ for (int i = start; i < vector.length; i++) {
+ // undo the old quantization
+ float v = (oldAlpha * vector[i]) + oldMinQuantile;
+ correction += quantizeFloat(v, null, 0);
+ }
+ return correction;
+ }
+
+ private float quantizeFloat(float v, byte[] dest, int destIndex) {
+ assert dest == null || destIndex < dest.length;
+ // Make sure the value is within the quantile range, cutting off the tails
+ // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -
+ // minQuantile)
+ float dx = v - minQuantile;
+ float dxc = Math.max(minQuantile, Math.min(maxQuantile, v)) - minQuantile;
+ // Scale the value to the range [0, 127], this is our quantized value
+ // scale = 127/(maxQuantile - minQuantile)
+ int roundedDxs = Math.round(scale * dxc);
+ // We multiply by `alpha` here to get the quantized value back into the original range
+ // to aid in calculating the corrective offset
+ float dxq = roundedDxs * alpha;
+ if (dest != null) {
+ dest[destIndex] = (byte) roundedDxs;
+ }
+ // Calculate the corrective offset that needs to be applied to the score
+ // in addition to the `byte * minQuantile * alpha` term in the equation
+ // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value
+ // will be rounded to the nearest whole number and lose some accuracy
+ // Additionally, we account for the global correction of `minQuantile^2` in the equation
+ return minQuantile * (v - minQuantile / 2.0F) + (dx - dxq) * dxq;
+ }
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java
index 82947d00343..0edeefbe353 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java
@@ -65,4 +65,39 @@ public interface VectorUtilSupport {
* @return the dot product
*/
long int4BitDotProduct(byte[] int4Quantized, byte[] binaryQuantized);
+
+ /**
+ * Quantizes {@code vector}, putting the result into {@code dest}.
+ *
+ * @param vector the vector to quantize
+ * @param dest the destination vector
+ * @param scale the scaling factor
+ * @param alpha the alpha value
+ * @param minQuantile the lower quantile of the distribution
+ * @param maxQuantile the upper quantile of the distribution
+ * @return the corrective offset that needs to be applied to the score
+ */
+ float minMaxScalarQuantize(
+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile);
+
+ /**
+ * Recalculates the offset for {@code vector}.
+ *
+ * @param vector the vector to quantize
+ * @param oldAlpha the previous alpha value
+ * @param oldMinQuantile the previous lower quantile
+ * @param scale the scaling factor
+ * @param alpha the alpha value
+ * @param minQuantile the lower quantile of the distribution
+ * @param maxQuantile the upper quantile of the distribution
+ * @return the new corrective offset
+ */
+ float recalculateScalarQuantizationOffset(
+ byte[] vector,
+ float oldAlpha,
+ float oldMinQuantile,
+ float scale,
+ float alpha,
+ float minQuantile,
+ float maxQuantile);
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
index 75998c2f7fb..a64c5165ddf 100644
--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java
@@ -334,4 +334,46 @@ public final class VectorUtil {
assert IntStream.range(0, to - 1).noneMatch(i -> buffer[i] > buffer[i + 1]);
return IMPL.findNextGEQ(buffer, target, from, to);
}
+
+ /**
+ * Scalar quantizes {@code vector}, putting the result into {@code dest}.
+ *
+ * @param vector the vector to quantize
+ * @param dest the destination vector
+ * @param scale the scaling factor
+ * @param alpha the alpha value
+ * @param minQuantile the lower quantile of the distribution
+ * @param maxQuantile the upper quantile of the distribution
+ * @return the corrective offset that needs to be applied to the score
+ */
+ public static float minMaxScalarQuantize(
+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {
+ if (vector.length != dest.length)
+ throw new IllegalArgumentException("source and destination arrays should be the same size");
+ return IMPL.minMaxScalarQuantize(vector, dest, scale, alpha, minQuantile, maxQuantile);
+ }
+
+ /**
+ * Recalculates the offset for {@code vector}.
+ *
+ * @param vector the vector to quantize
+ * @param oldAlpha the previous alpha value
+ * @param oldMinQuantile the previous lower quantile
+ * @param scale the scaling factor
+ * @param alpha the alpha value
+ * @param minQuantile the lower quantile of the distribution
+ * @param maxQuantile the upper quantile of the distribution
+ * @return the new corrective offset
+ */
+ public static float recalculateOffset(
+ byte[] vector,
+ float oldAlpha,
+ float oldMinQuantile,
+ float scale,
+ float alpha,
+ float minQuantile,
+ float maxQuantile) {
+ return IMPL.recalculateScalarQuantizationOffset(
+ vector, oldAlpha, oldMinQuantile, scale, alpha, minQuantile, maxQuantile);
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
index 34fb380a183..d6e2c0986a2 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
@@ -122,40 +122,15 @@ public class ScalarQuantizer {
public float quantize(float[] src, byte[] dest, VectorSimilarityFunction similarityFunction) {
assert src.length == dest.length;
assert similarityFunction != VectorSimilarityFunction.COSINE || VectorUtil.isUnitVector(src);
- float correction = 0;
- for (int i = 0; i < src.length; i++) {
- correction += quantizeFloat(src[i], dest, i);
- }
+
+ float correction =
+ VectorUtil.minMaxScalarQuantize(src, dest, scale, alpha, minQuantile, maxQuantile);
if (similarityFunction.equals(VectorSimilarityFunction.EUCLIDEAN)) {
return 0;
}
return correction;
}
- private float quantizeFloat(float v, byte[] dest, int destIndex) {
- assert dest == null || destIndex < dest.length;
- // Make sure the value is within the quantile range, cutting off the tails
- // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -
- // minQuantile)
- float dx = v - minQuantile;
- float dxc = Math.max(minQuantile, Math.min(maxQuantile, v)) - minQuantile;
- // Scale the value to the range [0, 127], this is our quantized value
- // scale = 127/(maxQuantile - minQuantile)
- float dxs = scale * dxc;
- // We multiply by `alpha` here to get the quantized value back into the original range
- // to aid in calculating the corrective offset
- float dxq = Math.round(dxs) * alpha;
- if (dest != null) {
- dest[destIndex] = (byte) Math.round(dxs);
- }
- // Calculate the corrective offset that needs to be applied to the score
- // in addition to the `byte * minQuantile * alpha` term in the equation
- // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value
- // will be rounded to the nearest whole number and lose some accuracy
- // Additionally, we account for the global correction of `minQuantile^2` in the equation
- return minQuantile * (v - minQuantile / 2.0F) + (dx - dxq) * dxq;
- }
-
/**
* Recalculate the old score corrective value given new current quantiles
*
@@ -171,13 +146,14 @@ public class ScalarQuantizer {
if (similarityFunction.equals(VectorSimilarityFunction.EUCLIDEAN)) {
return 0f;
}
- float correctiveOffset = 0f;
- for (int i = 0; i < quantizedVector.length; i++) {
- // dequantize the old value in order to recalculate the corrective offset
- float v = (oldQuantizer.alpha * quantizedVector[i]) + oldQuantizer.minQuantile;
- correctiveOffset += quantizeFloat(v, null, 0);
- }
- return correctiveOffset;
+ return VectorUtil.recalculateOffset(
+ quantizedVector,
+ oldQuantizer.alpha,
+ oldQuantizer.minQuantile,
+ scale,
+ alpha,
+ minQuantile,
+ maxQuantile);
}
/**
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
index b886611bba1..9aa4bc09e34 100644
--- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
+++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java
@@ -907,4 +907,98 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {
}
return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3);
}
+
+ @Override
+ public float minMaxScalarQuantize(
+ float[] vector, byte[] dest, float scale, float alpha, float minQuantile, float maxQuantile) {
+ assert vector.length == dest.length;
+ float correction = 0;
+ int i = 0;
+ // only vectorize if we have a viable BYTE_SPECIES we can use for output
+ if (VECTOR_BITSIZE >= 256) {
+ FloatVector sum = FloatVector.zero(FLOAT_SPECIES);
+
+ for (; i < FLOAT_SPECIES.loopBound(vector.length); i += FLOAT_SPECIES.length()) {
+ FloatVector v = FloatVector.fromArray(FLOAT_SPECIES, vector, i);
+
+ // Make sure the value is within the quantile range, cutting off the tails
+ // see first parenthesis in equation: byte = (float - minQuantile) * 127/(maxQuantile -
+ // minQuantile)
+ FloatVector dxc = v.min(maxQuantile).max(minQuantile).sub(minQuantile);
+ // Scale the value to the range [0, 127], this is our quantized value
+ // scale = 127/(maxQuantile - minQuantile)
+ // Math.round rounds to positive infinity, so do the same by +0.5 then truncating to int
+ Vector<Integer> roundedDxs =
+ fma(dxc, dxc.broadcast(scale), dxc.broadcast(0.5f)).convert(VectorOperators.F2I, 0);
+ // output this to the array
+ ((ByteVector) roundedDxs.castShape(BYTE_SPECIES, 0)).intoArray(dest, i);
+ // We multiply by `alpha` here to get the quantized value back into the original range
+ // to aid in calculating the corrective offset
+ FloatVector dxq = ((FloatVector) roundedDxs.castShape(FLOAT_SPECIES, 0)).mul(alpha);
+ // Calculate the corrective offset that needs to be applied to the score
+ // in addition to the `byte * minQuantile * alpha` term in the equation
+ // we add the `(dx - dxq) * dxq` term to account for the fact that the quantized value
+ // will be rounded to the nearest whole number and lose some accuracy
+ // Additionally, we account for the global correction of `minQuantile^2` in the equation
+ sum =
+ fma(
+ v.sub(minQuantile / 2f),
+ v.broadcast(minQuantile),
+ fma(v.sub(minQuantile).sub(dxq), dxq, sum));
+ }
+
+ correction = sum.reduceLanes(VectorOperators.ADD);
+ }
+
+ // complete the tail normally
+ correction +=
+ new DefaultVectorUtilSupport.ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)
+ .quantize(vector, dest, i);
+
+ return correction;
+ }
+
+ @Override
+ public float recalculateScalarQuantizationOffset(
+ byte[] vector,
+ float oldAlpha,
+ float oldMinQuantile,
+ float scale,
+ float alpha,
+ float minQuantile,
+ float maxQuantile) {
+ float correction = 0;
+ int i = 0;
+ // only vectorize if we have a viable BYTE_SPECIES that we can use
+ if (VECTOR_BITSIZE >= 256) {
+ FloatVector sum = FloatVector.zero(FLOAT_SPECIES);
+
+ for (; i < BYTE_SPECIES.loopBound(vector.length); i += BYTE_SPECIES.length()) {
+ FloatVector fv =
+ (FloatVector) ByteVector.fromArray(BYTE_SPECIES, vector, i).castShape(FLOAT_SPECIES, 0);
+ // undo the old quantization
+ FloatVector v = fma(fv, fv.broadcast(oldAlpha), fv.broadcast(oldMinQuantile));
+
+ // same operations as in quantize above
+ FloatVector dxc = v.min(maxQuantile).max(minQuantile).sub(minQuantile);
+ Vector<Integer> roundedDxs =
+ fma(dxc, dxc.broadcast(scale), dxc.broadcast(0.5f)).convert(VectorOperators.F2I, 0);
+ FloatVector dxq = ((FloatVector) roundedDxs.castShape(FLOAT_SPECIES, 0)).mul(alpha);
+ sum =
+ fma(
+ v.sub(minQuantile / 2f),
+ v.broadcast(minQuantile),
+ fma(v.sub(minQuantile).sub(dxq), dxq, sum));
+ }
+
+ correction = sum.reduceLanes(VectorOperators.ADD);
+ }
+
+ // complete the tail normally
+ correction +=
+ new DefaultVectorUtilSupport.ScalarQuantizer(alpha, scale, minQuantile, maxQuantile)
+ .recalculateOffset(vector, i, oldAlpha, oldMinQuantile);
+
+ return correction;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java
index 6443de752bf..ee62e839c06 100644
--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java
+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorUtilSupport.java
@@ -16,8 +16,13 @@
*/
package org.apache.lucene.internal.vectorization;
+import static org.hamcrest.Matchers.closeTo;
+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
import java.util.function.ToDoubleFunction;
import java.util.function.ToIntFunction;
import java.util.function.ToLongFunction;
@@ -25,17 +30,18 @@ import java.util.stream.IntStream;
public class TestVectorUtilSupport extends BaseVectorizationTestCase {
- private static final double DELTA = 1e-3;
-
private static final int[] VECTOR_SIZES = {
1, 4, 6, 8, 13, 16, 25, 32, 64, 100, 128, 207, 256, 300, 512, 702, 1024, 1536, 2046, 2048, 4096,
4098
};
private final int size;
+ private final double delta;
public TestVectorUtilSupport(int size) {
this.size = size;
+ // scale the delta with the size
+ this.delta = 1e-6 * size;
}
@ParametersFactory
@@ -164,11 +170,49 @@ public class TestVectorUtilSupport extends BaseVectorizationTestCase {
return packed;
}
+ public void testMinMaxScalarQuantize() {
+ Random r = random();
+ float min = r.nextFloat(-1, 1);
+ float max = r.nextFloat(min, 1);
+ float divisor = (float) ((1 << 7) - 1); // 7 bits quantization here
+
+ float scale = divisor / (max - min);
+ float alpha = (max - min) / divisor;
+
+ float[] vector = new float[size];
+ for (int i = 0; i < vector.length; i++) {
+ vector[i] = (r.nextFloat() * (max - min)) + min;
+ }
+
+ List<byte[]> outputs = new ArrayList<>();
+ assertFloatReturningProviders(
+ p -> {
+ byte[] output = new byte[size];
+ outputs.add(output);
+ return p.minMaxScalarQuantize(vector, output, scale, alpha, min, max);
+ });
+
+ // check the outputs are identical
+ for (int o = 1; o < outputs.size(); o++) {
+ assertArrayEquals(outputs.getFirst(), outputs.get(o));
+ }
+
+ // check recalculation too
+ float newMax = max * 2;
+ float newMin = min / 2;
+ float newScale = divisor / (newMax - newMin);
+ float newAlpha = (newMax - newMin) / divisor;
+
+ assertFloatReturningProviders(
+ p ->
+ p.recalculateScalarQuantizationOffset(
+ outputs.getFirst(), alpha, min, newScale, newAlpha, newMin, newMax));
+ }
+
private void assertFloatReturningProviders(ToDoubleFunction<VectorUtilSupport> func) {
- assertEquals(
- func.applyAsDouble(LUCENE_PROVIDER.getVectorUtilSupport()),
+ assertThat(
func.applyAsDouble(PANAMA_PROVIDER.getVectorUtilSupport()),
- DELTA);
+ closeTo(func.applyAsDouble(LUCENE_PROVIDER.getVectorUtilSupport()), delta));
}
private void assertIntReturningProviders(ToIntFunction<VectorUtilSupport> func) {