Skip to content

Apache Lucene open-source search software

Move legacy scalar formats to backwards_codecs (#15223)

Benjamin Trentec4cd9f66b12
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5946419d28a..06e7a1ca085 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -140,7 +140,9 @@ New Features
* GITHUB#15176: Add `[Float|Byte]VectorValues#rescorer(element[])` interface to allow optimized rescoring of vectors.
(Ben Trent)
-* GITHUB#15169: Add codecs for 4 and 8 bit Optimized Scalar Quantization vectors (Trevor McCulloch)
+* GITHUB#15169, GITHUB#15223: Add codecs for 4 and 8 bit Optimized Scalar Quantization vectors. The new format
+ `Lucene104HnswScalarQuantizedVectorsFormat` replaces the now legacy `Lucene99HnswScalarQuantizedVectorsFormat`
+ (Trevor McCulloch)
Improvements
---------------------
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 5c8cf801669..39d028232c5 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -59,7 +59,9 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat,
- org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat;
+ org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat,
+ org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat,
+ org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
provides org.apache.lucene.codecs.Codec with
org.apache.lucene.backward_codecs.lucene80.Lucene80Codec,
org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
similarity index 91%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
index 42e9381aa11..c1a0bba89b2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
@@ -29,9 +29,10 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.search.TaskExecutor;
import org.apache.lucene.util.hnsw.HnswGraph;
/**
@@ -42,10 +43,9 @@ import org.apache.lucene.util.hnsw.HnswGraph;
*
* @lucene.experimental
*/
-@Deprecated
public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {
- public static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat";
+ static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat";
/**
* Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
@@ -63,9 +63,6 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {
/** The format for storing, reading, merging vectors on disk */
private final FlatVectorsFormat flatVectorsFormat;
- private final int numMergeWorkers;
- private final TaskExecutor mergeExec;
-
/** Constructs a format using default graph construction parameters with 7 bit quantization */
public Lucene99HnswScalarQuantizedVectorsFormat() {
this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, 7, false, null, null);
@@ -129,25 +126,13 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {
throw new IllegalArgumentException(
"No executor service is needed as we'll use single thread to merge");
}
- this.numMergeWorkers = numMergeWorkers;
- if (mergeExec != null) {
- this.mergeExec = new TaskExecutor(mergeExec);
- } else {
- this.mergeExec = null;
- }
this.flatVectorsFormat =
new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);
}
@Override
public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene99HnswVectorsWriter(
- state,
- maxConn,
- beamWidth,
- flatVectorsFormat.fieldsWriter(state),
- numMergeWorkers,
- mergeExec);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
similarity index 91%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
index 39d8416a1ab..b7f7e16c5cf 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
@@ -23,6 +23,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -31,15 +32,15 @@ import org.apache.lucene.index.SegmentWriteState;
*
* @lucene.experimental
*/
-@Deprecated
public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
// The bits that are allowed for scalar quantization
// We only allow unsigned byte (8), signed byte (7), and half-byte (4)
private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4);
- public static final String QUANTIZED_VECTOR_COMPONENT = "QVEC";
+ static final String QUANTIZED_VECTOR_COMPONENT = "QVEC";
+ static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
- public static final String NAME = "Lucene99ScalarQuantizedVectorsFormat";
+ static final String NAME = "Lucene99ScalarQuantizedVectorsFormat";
static final int VERSION_START = 0;
static final int VERSION_ADD_BITS = 1;
@@ -49,7 +50,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
static final String META_EXTENSION = "vemq";
static final String VECTOR_DATA_EXTENSION = "veq";
- private static final FlatVectorsFormat rawVectorFormat =
+ static final FlatVectorsFormat rawVectorFormat =
new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
/** The minimum confidence interval */
@@ -119,7 +120,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
this.flatVectorScorer = FlatVectorScorerUtil.getLucene99ScalarQuantizedVectorsScorer();
}
- public static float calculateDefaultConfidenceInterval(int vectorDimension) {
+ static float calculateDefaultConfidenceInterval(int vectorDimension) {
return Math.max(MINIMUM_CONFIDENCE_INTERVAL, 1f - (1f / (vectorDimension + 1)));
}
@@ -143,13 +144,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene99ScalarQuantizedVectorsWriter(
- state,
- confidenceInterval,
- bits,
- compress,
- rawVectorFormat.fieldsWriter(state),
- flatVectorScorer);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index c9dbc1edc6e..e3e092da53b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding;
-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;
import java.io.IOException;
import java.util.Map;
@@ -57,7 +57,7 @@ import org.apache.lucene.util.quantization.ScalarQuantizer;
*
* @lucene.experimental
*/
-public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReader
+final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReader
implements QuantizedVectorsReader {
private static final long SHALLOW_SIZE =
@@ -68,7 +68,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
private final FlatVectorsReader rawVectorsReader;
private final FieldInfos fieldInfos;
- public Lucene99ScalarQuantizedVectorsReader(
+ Lucene99ScalarQuantizedVectorsReader(
SegmentReadState state, FlatVectorsReader rawVectorsReader, FlatVectorsScorer scorer)
throws IOException {
super(scorer);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
index 051c926a679..22fe9f2c569 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -38,20 +38,20 @@ import org.apache.lucene.util.quantization.ScalarQuantizer;
*/
public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectorValues {
- protected final int dimension;
- protected final int size;
- protected final int numBytes;
- protected final ScalarQuantizer scalarQuantizer;
- protected final VectorSimilarityFunction similarityFunction;
- protected final FlatVectorsScorer vectorsScorer;
- protected final boolean compress;
-
- protected final IndexInput slice;
- protected final byte[] binaryValue;
- protected final ByteBuffer byteBuffer;
- protected final int byteSize;
- protected int lastOrd = -1;
- protected final float[] scoreCorrectionConstant = new float[1];
+ final int dimension;
+ final int size;
+ final int numBytes;
+ final ScalarQuantizer scalarQuantizer;
+ final VectorSimilarityFunction similarityFunction;
+ final FlatVectorsScorer vectorsScorer;
+ final boolean compress;
+
+ final IndexInput slice;
+ final byte[] binaryValue;
+ final ByteBuffer byteBuffer;
+ final int byteSize;
+ int lastOrd = -1;
+ final float[] scoreCorrectionConstant = new float[1];
static void decompressBytes(byte[] compressed, int numBytes) {
if (numBytes == compressed.length) {
@@ -159,7 +159,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
return numBytes;
}
- public static OffHeapQuantizedByteVectorValues load(
+ static OffHeapQuantizedByteVectorValues load(
OrdToDocDISIReaderConfiguration configuration,
int dimension,
int size,
@@ -206,6 +206,17 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
*/
public static class DenseOffHeapVectorValues extends OffHeapQuantizedByteVectorValues {
+ /**
+ * Create dense off-heap vector values
+ *
+ * @param dimension vector dimension
+ * @param size number of vectors
+ * @param scalarQuantizer the scalar quantizer
+ * @param compress whether the vectors are compressed
+ * @param similarityFunction the similarity function
+ * @param vectorsScorer the vectors scorer
+ * @param slice the index input slice containing the vector data
+ */
public DenseOffHeapVectorValues(
int dimension,
int size,
@@ -266,7 +277,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
private final IndexInput dataIn;
private final OrdToDocDISIReaderConfiguration configuration;
- public SparseOffHeapVectorValues(
+ SparseOffHeapVectorValues(
OrdToDocDISIReaderConfiguration configuration,
int dimension,
int size,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.java
index 71eba0b43bf..8e6484cae70 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.nio.ByteBuffer;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index 18b19d56c2c..f8b18e7e539 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -18,3 +18,5 @@ org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat
+org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat
+org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..bfd09fbc8b2
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.backward_codecs.lucene99;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.index.SegmentWriteState;
+
+public class Lucene99RWScalarQuantizedVectorsFormat extends Lucene99ScalarQuantizedVectorsFormat {
+ /** Sole constructor */
+ protected Lucene99RWScalarQuantizedVectorsFormat() {
+ super();
+ }
+
+ Lucene99RWScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boolean compress) {
+ super(confidenceInterval, bits, compress);
+ }
+
+ @Override
+ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new Lucene99ScalarQuantizedVectorsWriter(
+ state,
+ confidenceInterval,
+ bits,
+ compress,
+ rawVectorFormat.fieldsWriter(state),
+ flatVectorScorer);
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.java
similarity index 87%
rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.java
index 8eac95dd9ef..a04863bf505 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.java
@@ -24,20 +24,18 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter;
import org.apache.lucene.index.SegmentWriteState;
-class Lucene99RWHnswScalarQuantizationVectorsFormat
+/** This is used to test an older version of Lucene99ScalarQuantizedVectorsWriter */
+class Lucene99RWV0HnswScalarQuantizationVectorsFormat
extends Lucene99HnswScalarQuantizedVectorsFormat {
private final FlatVectorsFormat flatVectorsFormat = new Lucene99RWScalarQuantizedFormat();
/** Sole constructor */
- protected Lucene99RWHnswScalarQuantizationVectorsFormat() {
+ protected Lucene99RWV0HnswScalarQuantizationVectorsFormat() {
super();
}
@@ -58,6 +56,7 @@ class Lucene99RWHnswScalarQuantizationVectorsFormat
@Override
public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ // this forces an older version of the writer
return new Lucene99ScalarQuantizedVectorsWriter(
state,
null,
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.java
new file mode 100644
index 00000000000..497f14290ef
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.backward_codecs.lucene99;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
+import org.apache.lucene.index.SegmentWriteState;
+
+class Lucene99RWV1HnswScalarQuantizationVectorsFormat
+ extends Lucene99HnswScalarQuantizedVectorsFormat {
+
+ private final FlatVectorsFormat flatVectorsFormat;
+
+ /** Sole constructor */
+ protected Lucene99RWV1HnswScalarQuantizationVectorsFormat() {
+ super();
+ this.flatVectorsFormat = new Lucene99RWScalarQuantizedVectorsFormat();
+ }
+
+ public Lucene99RWV1HnswScalarQuantizationVectorsFormat(
+ int maxConn,
+ int beamWidth,
+ int numMergeWorkers,
+ int bits,
+ boolean compress,
+ Float confidenceInterval,
+ ExecutorService mergeExec) {
+ super(maxConn, beamWidth, numMergeWorkers, bits, compress, confidenceInterval, mergeExec);
+ this.flatVectorsFormat =
+ new Lucene99RWScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);
+ }
+
+ @Override
+ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new Lucene99HnswVectorsWriter(
+ state,
+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+ flatVectorsFormat.fieldsWriter(state),
+ 1,
+ null);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index cf960672504..d88e2d121d9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -15,13 +15,13 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval;
import static org.apache.lucene.codecs.KnnVectorsWriter.MergedVectorValues.hasVectorValues;
-import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;
-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT;
-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
index 3ad2cab8869..1b23e1108d9 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import static java.lang.String.format;
import static org.hamcrest.Matchers.is;
@@ -28,6 +28,8 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -79,7 +81,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
}
private final KnnVectorsFormat getKnnFormat(int bits) {
- return new Lucene99HnswScalarQuantizedVectorsFormat(
+ return new Lucene99RWV1HnswScalarQuantizationVectorsFormat(
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
1,
@@ -186,7 +188,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
newIndexWriterConfig()
.setCodec(
TestUtil.alwaysKnnVectorsFormat(
- new Lucene99HnswScalarQuantizedVectorsFormat(
+ new Lucene99RWV1HnswScalarQuantizationVectorsFormat(
16, 100, 1, (byte) 7, false, 0.9f, null))))) {
for (float[] vector : vectors) {
Document doc = new Document();
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
index d7af766ee07..f16c95c92e1 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
@@ -37,7 +37,7 @@ import org.apache.lucene.tests.util.TestUtil;
public class TestLucene99HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
@Override
protected Codec getCodec() {
- return TestUtil.alwaysKnnVectorsFormat(new Lucene99RWHnswScalarQuantizationVectorsFormat());
+ return TestUtil.alwaysKnnVectorsFormat(new Lucene99RWV0HnswScalarQuantizationVectorsFormat());
}
public void testSimpleOffHeapSize() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
similarity index 96%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
index 778111fc1f0..71fbff929ab 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
-import static org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes;
+import static org.apache.lucene.backward_codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -26,6 +26,9 @@ import java.nio.ByteOrder;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -53,7 +56,7 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
private static Codec getCodec(int bits, boolean compress) {
return TestUtil.alwaysKnnVectorsFormat(
- new Lucene99HnswScalarQuantizedVectorsFormat(
+ new Lucene99RWV1HnswScalarQuantizationVectorsFormat(
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
1,
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
index 7156afd9cc3..609d2496d54 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
@@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import static java.lang.String.format;
-import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.oneOf;
@@ -71,8 +71,8 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsForm
confidenceInterval = 0f;
}
format =
- new Lucene99ScalarQuantizedVectorsFormat(
- confidenceInterval, bits, bits == 4 ? random().nextBoolean() : false);
+ new Lucene99RWScalarQuantizedVectorsFormat(
+ confidenceInterval, bits, bits == 4 && random().nextBoolean());
super.setUp();
}
@@ -83,7 +83,7 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsForm
private Codec getCodec(float confidenceInterval) {
return TestUtil.alwaysKnnVectorsFormat(
- new Lucene99ScalarQuantizedVectorsFormat(
+ new Lucene99RWScalarQuantizedVectorsFormat(
confidenceInterval, bits, bits == 4 ? random().nextBoolean() : false));
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java
index 0bf2a4ef6b8..d51578e957e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
index 384ce889c5a..5858eab0e88 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
@@ -22,7 +22,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -68,7 +68,7 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTe
protected Codec getCodec() {
return TestUtil.alwaysKnnVectorsFormat(
- new Lucene99HnswScalarQuantizedVectorsFormat(
+ new Lucene104HnswScalarQuantizedVectorsFormat(
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH));
}
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 0952f269686..108f8bcadc6 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -84,8 +84,6 @@ module org.apache.lucene.core {
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat,
- org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat,
- org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java
index 4907ad67cd4..54857395c4b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java
@@ -21,7 +21,6 @@
* storage formats or scoring without significant changes to the HNSW code. Some examples for
* scoring include {@link org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer} and {@link
* org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer}. Some examples for storing include {@link
- * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat} and {@link
- * org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat}.
+ * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat}.
*/
package org.apache.lucene.codecs.hnsw;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
index 5f45c8cabd8..a2072a29788 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
@@ -39,6 +39,13 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
this.nonQuantizedDelegate = nonQuantizedDelegate;
}
+ static void checkDimensions(int queryLen, int fieldLen) {
+ if (queryLen != fieldLen) {
+ throw new IllegalArgumentException(
+ "vector query dimension: " + queryLen + " differs from field dimension: " + fieldLen);
+ }
+ }
+
@Override
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
@@ -55,6 +62,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
throws IOException {
if (vectorValues instanceof QuantizedByteVectorValues qv) {
+ checkDimensions(target.length, qv.dimension());
OptimizedScalarQuantizer quantizer = qv.getQuantizer();
byte[] targetQuantized =
new byte
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
index 9acb43c64cf..44e1ef092c1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
@@ -130,6 +130,15 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
*/
SEVEN_BIT(2, (byte) 7, 1);
+ public static ScalarEncoding fromNumBits(int bits) {
+ for (ScalarEncoding encoding : values()) {
+ if (encoding.bits == bits) {
+ return encoding;
+ }
+ }
+ throw new IllegalArgumentException("No encoding for " + bits + " bits");
+ }
+
/** The number used to identify this encoding on the wire, rather than relying on ordinal. */
private final int wireNumber;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 0251224998b..8012d6095c2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -53,9 +53,12 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedVectorsReader;
+import org.apache.lucene.util.quantization.ScalarQuantizer;
/** Reader for scalar quantized vectors in the Lucene 10.4 format. */
-class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {
+class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
+ implements QuantizedVectorsReader {
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Lucene104ScalarQuantizedVectorsReader.class);
@@ -339,6 +342,64 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {
return FieldEntry.create(input, vectorEncoding, info.getVectorSimilarityFunction());
}
+ @Override
+ public org.apache.lucene.util.quantization.QuantizedByteVectorValues getQuantizedVectorValues(
+ String field) throws IOException {
+ FieldEntry fi = fields.get(field);
+ if (fi == null) {
+ return null;
+ }
+ if (fi.vectorEncoding != VectorEncoding.FLOAT32) {
+ throw new IllegalArgumentException(
+ "field=\""
+ + field
+ + "\" is encoded as: "
+ + fi.vectorEncoding
+ + " expected: "
+ + VectorEncoding.FLOAT32);
+ }
+ var qv =
+ OffHeapScalarQuantizedVectorValues.load(
+ fi.ordToDocDISIReaderConfiguration,
+ fi.dimension,
+ fi.size,
+ new OptimizedScalarQuantizer(fi.similarityFunction),
+ fi.scalarEncoding,
+ fi.similarityFunction,
+ vectorScorer,
+ fi.centroid,
+ fi.centroidDP,
+ fi.vectorDataOffset,
+ fi.vectorDataLength,
+ quantizedVectorData);
+ return new org.apache.lucene.util.quantization.QuantizedByteVectorValues() {
+ @Override
+ public float getScoreCorrectionConstant(int ord) throws IOException {
+ return 0;
+ }
+
+ @Override
+ public byte[] vectorValue(int ord) throws IOException {
+ return qv.vectorValue(ord);
+ }
+
+ @Override
+ public int dimension() {
+ return qv.dimension();
+ }
+
+ @Override
+ public int size() {
+ return qv.size();
+ }
+ };
+ }
+
+ @Override
+ public ScalarQuantizer getQuantizationState(String fieldName) {
+ return null;
+ }
+
private record FieldEntry(
VectorSimilarityFunction similarityFunction,
VectorEncoding vectorEncoding,
@@ -444,6 +505,11 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {
return quantizedVectorValues.scorer(query);
}
+ @Override
+ public VectorScorer rescorer(float[] target) throws IOException {
+ return rawVectorValues.rescorer(target);
+ }
+
QuantizedByteVectorValues getQuantizedVectorValues() throws IOException {
return quantizedVectorValues;
}
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index fde541c2ac0..c614014dec6 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -14,8 +14,6 @@
# limitations under the License.
org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat
-org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat
-org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat
org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat
org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat
org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
index cd50705ac73..7b05a018afa 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
@@ -30,7 +30,7 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -85,7 +85,7 @@ public class TestKnnGraph extends LuceneTestCase {
codec =
TestUtil.alwaysKnnVectorsFormat(
quantized
- ? new Lucene99HnswScalarQuantizedVectorsFormat(
+ ? new Lucene104HnswScalarQuantizedVectorsFormat(
M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH)
: new Lucene99HnswVectorsFormat(M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH));
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
index b849bd2a8fc..1c63ef4bf15 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
@@ -22,7 +22,8 @@ import java.util.List;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -50,18 +51,13 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTe
private static final int VECTOR_DIMENSION = 8;
KnnVectorsFormat format;
- Float confidenceInterval;
int bits;
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- bits = random().nextBoolean() ? 4 : 7;
- confidenceInterval = random().nextBoolean() ? random().nextFloat(0.90f, 1.0f) : null;
- if (random().nextBoolean()) {
- confidenceInterval = 0f;
- }
+ bits = random().nextBoolean() ? 4 : 8;
format = getKnnFormat(bits);
savedCodec = Codec.getDefault();
Codec.setDefault(getCodec());
@@ -77,14 +73,12 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTe
return TestUtil.alwaysKnnVectorsFormat(format);
}
- private final KnnVectorsFormat getKnnFormat(int bits) {
- return new Lucene99HnswScalarQuantizedVectorsFormat(
+ private KnnVectorsFormat getKnnFormat(int bits) {
+ return new Lucene104HnswScalarQuantizedVectorsFormat(
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.fromNumBits(bits),
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
1,
- bits,
- bits == 4 ? random().nextBoolean() : false,
- confidenceInterval,
null);
}
@@ -131,11 +125,11 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTe
w.flush();
}
- // index some 7 bit quantized vectors
+ // index some 8 bit quantized vectors
try (IndexWriter w =
new IndexWriter(
dir,
- newIndexWriterConfig().setCodec(TestUtil.alwaysKnnVectorsFormat(getKnnFormat(7))))) {
+ newIndexWriterConfig().setCodec(TestUtil.alwaysKnnVectorsFormat(getKnnFormat(8))))) {
for (int j = 0; j < numSegments; j++) {
for (int i = 0; i < numVectors; i++) {
Document doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java
index e7dea987816..0e4db8679e7 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
@@ -66,7 +66,7 @@ public class TestRescoreTopNQuery extends LuceneTestCase {
// Set up the IndexWriterConfig to use quantized vector storage
config = new IndexWriterConfig();
config.setCodec(
- TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat()));
+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));
}
@Test
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
index 1f6c5d525e4..653c519729a 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinWorkerThread;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.StoredField;
@@ -63,7 +63,7 @@ public class TestBpVectorReorderer extends LuceneTestCase {
private void createQuantizedIndex(Directory dir, List<float[]> vectors) throws IOException {
IndexWriterConfig cfg = new IndexWriterConfig();
cfg.setCodec(
- TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat(8, 32)));
+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat(8, 32)));
try (IndexWriter writer = new IndexWriter(dir, cfg)) {
int i = 0;
for (float[] vector : vectors) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
index 6539423e702..420d6d0164e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
@@ -33,10 +33,11 @@ import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;
import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.codecs.memory.FSTPostingsFormat;
@@ -272,32 +273,24 @@ public class RandomCodec extends AssertingCodec {
TestUtil.nextInt(random, 10, 50),
concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
- new Lucene99HnswScalarQuantizedVectorsFormat(
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT,
+ TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,
+ TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.PACKED_NIBBLE,
TestUtil.nextInt(random, 5, 50),
TestUtil.nextInt(random, 10, 50),
concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
- 7,
- false,
- randomConfidenceInterval(random),
concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
- // TODO: also test 4-bit quantization, but this must somehow be restricted to even-length
- // fields
- /*
- * new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),
- * TestUtil.nextInt(random, 10, 50),
- * 1,
- * 4,
- * random.nextBoolean(),
- * randomConfidenceInterval(random),
- * null),
- * new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),
- * TestUtil.nextInt(random, 10, 50),
- * TestUtil.nextInt(random, 2, 8),
- * 4,
- * random.nextBoolean(),
- * randomConfidenceInterval(random),
- * ForkJoinPool.commonPool()),
- */
new AssertingKnnVectorsFormat());
Collections.shuffle(formats, random);
@@ -316,18 +309,6 @@ public class RandomCodec extends AssertingCodec {
}
}
- private final Float randomConfidenceInterval(Random random) {
- switch (random.nextInt(3)) {
- default:
- case 0:
- return null;
- case 1:
- return 0f;
- case 2:
- return random.nextFloat(0.9f, 1f);
- }
- }
-
public RandomCodec(Random random) {
this(random, Collections.<String>emptySet());
}