Apache Lucene open-source search software
Move legacy scalar formats to backwards_codecs (#15223)
Benjamin Trentec4cd9f66b12
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5946419d28a..06e7a1ca085 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -140,7 +140,9 @@ New Features* GITHUB#15176: Add `[Float|Byte]VectorValues#rescorer(element[])` interface to allow optimized rescoring of vectors.(Ben Trent)-* GITHUB#15169: Add codecs for 4 and 8 bit Optimized Scalar Quantization vectors (Trevor McCulloch)+* GITHUB#15169, GITHUB#15223: Add codecs for 4 and 8 bit Optimized Scalar Quantization vectors. The new format+ `Lucene104HnswScalarQuantizedVectorsFormat` replaces the now legacy `Lucene99HnswScalarQuantizedVectorsFormat`+ (Trevor McCulloch)Improvements---------------------diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.javaindex 5c8cf801669..39d028232c5 100644--- a/lucene/backward-codecs/src/java/module-info.java+++ b/lucene/backward-codecs/src/java/module-info.java@@ -59,7 +59,9 @@ module org.apache.lucene.backward_codecs {org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat,org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat,- org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat;+ org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat,+ org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat,+ org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;provides org.apache.lucene.codecs.Codec withorg.apache.lucene.backward_codecs.lucene80.Lucene80Codec,org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.javasimilarity index 91%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.javaindex 42e9381aa11..c1a0bba89b2 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;@@ -29,9 +29,10 @@ import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.KnnVectorsWriter;import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.search.TaskExecutor;import org.apache.lucene.util.hnsw.HnswGraph;/**@@ -42,10 +43,9 @@ import org.apache.lucene.util.hnsw.HnswGraph;** @lucene.experimental*/-@Deprecatedpublic class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {- public static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat";+ static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat";/*** Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to@@ -63,9 +63,6 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {/** The format for storing, reading, merging vectors on disk */private final FlatVectorsFormat flatVectorsFormat;- private final int numMergeWorkers;- private final TaskExecutor mergeExec;-/** Constructs a format using default graph construction parameters with 7 bit quantization */public Lucene99HnswScalarQuantizedVectorsFormat() {this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, 7, false, null, null);@@ -129,25 +126,13 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {throw new IllegalArgumentException("No executor service is needed as we'll use single thread to merge");}- this.numMergeWorkers = numMergeWorkers;- if (mergeExec != null) {- this.mergeExec = new TaskExecutor(mergeExec);- } else {- this.mergeExec = null;- }this.flatVectorsFormat =new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);}@Overridepublic KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new Lucene99HnswVectorsWriter(- state,- maxConn,- beamWidth,- flatVectorsFormat.fieldsWriter(state),- numMergeWorkers,- mergeExec);+ throw new UnsupportedOperationException("Old codecs may only be used for reading");}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.javasimilarity index 91%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.javaindex 39d8416a1ab..b7f7e16c5cf 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import java.io.IOException;import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;@@ -23,6 +23,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -31,15 +32,15 @@ import org.apache.lucene.index.SegmentWriteState;** @lucene.experimental*/-@Deprecatedpublic class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {// The bits that are allowed for scalar quantization// We only allow unsigned byte (8), signed byte (7), and half-byte (4)private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4);- public static final String QUANTIZED_VECTOR_COMPONENT = "QVEC";+ static final String QUANTIZED_VECTOR_COMPONENT = "QVEC";+ static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;- public static final String NAME = "Lucene99ScalarQuantizedVectorsFormat";+ static final String NAME = "Lucene99ScalarQuantizedVectorsFormat";static final int VERSION_START = 0;static final int VERSION_ADD_BITS = 1;@@ -49,7 +50,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {static final String META_EXTENSION = "vemq";static final String VECTOR_DATA_EXTENSION = "veq";- private static final FlatVectorsFormat rawVectorFormat =+ static final FlatVectorsFormat rawVectorFormat =new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());/** The minimum confidence interval */@@ -119,7 +120,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {this.flatVectorScorer = FlatVectorScorerUtil.getLucene99ScalarQuantizedVectorsScorer();}- public static float calculateDefaultConfidenceInterval(int vectorDimension) {+ static float calculateDefaultConfidenceInterval(int vectorDimension) {return Math.max(MINIMUM_CONFIDENCE_INTERVAL, 1f - (1f / (vectorDimension + 1)));}@@ -143,13 +144,7 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {@Overridepublic FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new Lucene99ScalarQuantizedVectorsWriter(- state,- confidenceInterval,- bits,- compress,- rawVectorFormat.fieldsWriter(state),- flatVectorScorer);+ throw new UnsupportedOperationException("Old codecs may only be used for reading");}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex c9dbc1edc6e..e3e092da53b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -15,11 +15,11 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction;import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding;-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;import java.io.IOException;import java.util.Map;@@ -57,7 +57,7 @@ import org.apache.lucene.util.quantization.ScalarQuantizer;** @lucene.experimental*/-public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReader+final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReaderimplements QuantizedVectorsReader {private static final long SHALLOW_SIZE =@@ -68,7 +68,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadeprivate final FlatVectorsReader rawVectorsReader;private final FieldInfos fieldInfos;- public Lucene99ScalarQuantizedVectorsReader(+ Lucene99ScalarQuantizedVectorsReader(SegmentReadState state, FlatVectorsReader rawVectorsReader, FlatVectorsScorer scorer)throws IOException {super(scorer);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.javasimilarity index 92%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.javaindex 051c926a679..22fe9f2c569 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import java.io.IOException;import java.nio.ByteBuffer;@@ -38,20 +38,20 @@ import org.apache.lucene.util.quantization.ScalarQuantizer;*/public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectorValues {- protected final int dimension;- protected final int size;- protected final int numBytes;- protected final ScalarQuantizer scalarQuantizer;- protected final VectorSimilarityFunction similarityFunction;- protected final FlatVectorsScorer vectorsScorer;- protected final boolean compress;-- protected final IndexInput slice;- protected final byte[] binaryValue;- protected final ByteBuffer byteBuffer;- protected final int byteSize;- protected int lastOrd = -1;- protected final float[] scoreCorrectionConstant = new float[1];+ final int dimension;+ final int size;+ final int numBytes;+ final ScalarQuantizer scalarQuantizer;+ final VectorSimilarityFunction similarityFunction;+ final FlatVectorsScorer vectorsScorer;+ final boolean compress;++ final IndexInput slice;+ final byte[] binaryValue;+ final ByteBuffer byteBuffer;+ final int byteSize;+ int lastOrd = -1;+ final float[] scoreCorrectionConstant = new float[1];static void decompressBytes(byte[] compressed, int numBytes) {if (numBytes == compressed.length) {@@ -159,7 +159,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectreturn numBytes;}- public static OffHeapQuantizedByteVectorValues load(+ static OffHeapQuantizedByteVectorValues load(OrdToDocDISIReaderConfiguration configuration,int dimension,int size,@@ -206,6 +206,17 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect*/public static class DenseOffHeapVectorValues extends OffHeapQuantizedByteVectorValues {+ /**+ * Create dense off-heap vector values+ *+ * @param dimension vector dimension+ * @param size number of vectors+ * @param scalarQuantizer the scalar quantizer+ * @param compress whether the vectors are compressed+ * @param similarityFunction the similarity function+ * @param vectorsScorer the vectors scorer+ * @param slice the index input slice containing the vector data+ */public DenseOffHeapVectorValues(int dimension,int size,@@ -266,7 +277,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectprivate final IndexInput dataIn;private final OrdToDocDISIReaderConfiguration configuration;- public SparseOffHeapVectorValues(+ SparseOffHeapVectorValues(OrdToDocDISIReaderConfiguration configuration,int dimension,int size,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.javasimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.javaindex 71eba0b43bf..8e6484cae70 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedFloatVectorValues.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedFloatVectorValues.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import java.io.IOException;import java.nio.ByteBuffer;diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormatindex 18b19d56c2c..f8b18e7e539 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat@@ -18,3 +18,5 @@ org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormatorg.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormatorg.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormatorg.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat+org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat+org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormatdiff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.javanew file mode 100644index 00000000000..bfd09fbc8b2--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWScalarQuantizedVectorsFormat.java@@ -0,0 +1,44 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.backward_codecs.lucene99;++import java.io.IOException;+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;+import org.apache.lucene.index.SegmentWriteState;++public class Lucene99RWScalarQuantizedVectorsFormat extends Lucene99ScalarQuantizedVectorsFormat {+ /** Sole constructor */+ protected Lucene99RWScalarQuantizedVectorsFormat() {+ super();+ }++ Lucene99RWScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boolean compress) {+ super(confidenceInterval, bits, compress);+ }++ @Override+ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {+ return new Lucene99ScalarQuantizedVectorsWriter(+ state,+ confidenceInterval,+ bits,+ compress,+ rawVectorFormat.fieldsWriter(state),+ flatVectorScorer);+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.javasimilarity index 87%rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.javaindex 8eac95dd9ef..a04863bf505 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV0HnswScalarQuantizationVectorsFormat.java@@ -24,20 +24,18 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer;import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;-import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;-import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter;import org.apache.lucene.index.SegmentWriteState;-class Lucene99RWHnswScalarQuantizationVectorsFormat+/** This is used to test an older version of Lucene99ScalarQuantizedVectorsWriter */+class Lucene99RWV0HnswScalarQuantizationVectorsFormatextends Lucene99HnswScalarQuantizedVectorsFormat {private final FlatVectorsFormat flatVectorsFormat = new Lucene99RWScalarQuantizedFormat();/** Sole constructor */- protected Lucene99RWHnswScalarQuantizationVectorsFormat() {+ protected Lucene99RWV0HnswScalarQuantizationVectorsFormat() {super();}@@ -58,6 +56,7 @@ class Lucene99RWHnswScalarQuantizationVectorsFormat@Overridepublic FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {+ // this forces an older version of the writerreturn new Lucene99ScalarQuantizedVectorsWriter(state,null,diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.javanew file mode 100644index 00000000000..497f14290ef--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWV1HnswScalarQuantizationVectorsFormat.java@@ -0,0 +1,62 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.backward_codecs.lucene99;++import java.io.IOException;+import java.util.concurrent.ExecutorService;+import org.apache.lucene.codecs.KnnVectorsWriter;+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;+import org.apache.lucene.index.SegmentWriteState;++class Lucene99RWV1HnswScalarQuantizationVectorsFormat+ extends Lucene99HnswScalarQuantizedVectorsFormat {++ private final FlatVectorsFormat flatVectorsFormat;++ /** Sole constructor */+ protected Lucene99RWV1HnswScalarQuantizationVectorsFormat() {+ super();+ this.flatVectorsFormat = new Lucene99RWScalarQuantizedVectorsFormat();+ }++ public Lucene99RWV1HnswScalarQuantizationVectorsFormat(+ int maxConn,+ int beamWidth,+ int numMergeWorkers,+ int bits,+ boolean compress,+ Float confidenceInterval,+ ExecutorService mergeExec) {+ super(maxConn, beamWidth, numMergeWorkers, bits, compress, confidenceInterval, mergeExec);+ this.flatVectorsFormat =+ new Lucene99RWScalarQuantizedVectorsFormat(confidenceInterval, bits, compress);+ }++ @Override+ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {+ return new Lucene99HnswVectorsWriter(+ state,+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,+ flatVectorsFormat.fieldsWriter(state),+ 1,+ null);+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javaindex cf960672504..d88e2d121d9 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java@@ -15,13 +15,13 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval;import static org.apache.lucene.codecs.KnnVectorsWriter.MergedVectorValues.hasVectorValues;-import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL;-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT;-import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.javasimilarity index 97%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.javaindex 3ad2cab8869..1b23e1108d9 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import static java.lang.String.format;import static org.hamcrest.Matchers.is;@@ -28,6 +28,8 @@ import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.FilterCodec;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;@@ -79,7 +81,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat}private final KnnVectorsFormat getKnnFormat(int bits) {- return new Lucene99HnswScalarQuantizedVectorsFormat(+ return new Lucene99RWV1HnswScalarQuantizationVectorsFormat(Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,1,@@ -186,7 +188,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormatnewIndexWriterConfig().setCodec(TestUtil.alwaysKnnVectorsFormat(- new Lucene99HnswScalarQuantizedVectorsFormat(+ new Lucene99RWV1HnswScalarQuantizationVectorsFormat(16, 100, 1, (byte) 7, false, 0.9f, null))))) {for (float[] vector : vectors) {Document doc = new Document();diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.javaindex d7af766ee07..f16c95c92e1 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java@@ -37,7 +37,7 @@ import org.apache.lucene.tests.util.TestUtil;public class TestLucene99HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {@Overrideprotected Codec getCodec() {- return TestUtil.alwaysKnnVectorsFormat(new Lucene99RWHnswScalarQuantizationVectorsFormat());+ return TestUtil.alwaysKnnVectorsFormat(new Lucene99RWV0HnswScalarQuantizationVectorsFormat());}public void testSimpleOffHeapSize() throws IOException {diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.javasimilarity index 96%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.javaindex 778111fc1f0..71fbff929ab 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java@@ -15,9 +15,9 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;-import static org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes;+import static org.apache.lucene.backward_codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes;import java.io.ByteArrayOutputStream;import java.io.IOException;@@ -26,6 +26,9 @@ import java.nio.ByteOrder;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;+import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -53,7 +56,7 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {private static Codec getCodec(int bits, boolean compress) {return TestUtil.alwaysKnnVectorsFormat(- new Lucene99HnswScalarQuantizedVectorsFormat(+ new Lucene99RWV1HnswScalarQuantizationVectorsFormat(Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,1,diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.javasimilarity index 97%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.javaindex 7156afd9cc3..609d2496d54 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java@@ -14,10 +14,10 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import static java.lang.String.format;-import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import static org.hamcrest.Matchers.is;import static org.hamcrest.Matchers.oneOf;@@ -71,8 +71,8 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormconfidenceInterval = 0f;}format =- new Lucene99ScalarQuantizedVectorsFormat(- confidenceInterval, bits, bits == 4 ? random().nextBoolean() : false);+ new Lucene99RWScalarQuantizedVectorsFormat(+ confidenceInterval, bits, bits == 4 && random().nextBoolean());super.setUp();}@@ -83,7 +83,7 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormprivate Codec getCodec(float confidenceInterval) {return TestUtil.alwaysKnnVectorsFormat(- new Lucene99ScalarQuantizedVectorsFormat(+ new Lucene99RWScalarQuantizedVectorsFormat(confidenceInterval, bits, bits == 4 ? random().nextBoolean() : false));}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.javasimilarity index 98%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.javaindex 0bf2a4ef6b8..d51578e957e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsWriter.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.lucene.codecs.lucene99;+package org.apache.lucene.backward_codecs.lucene99;import java.io.IOException;import java.util.ArrayList;diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.javaindex 384ce889c5a..5858eab0e88 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java@@ -22,7 +22,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.IOException;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.KnnVectorsReader;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -68,7 +68,7 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTeprotected Codec getCodec() {return TestUtil.alwaysKnnVectorsFormat(- new Lucene99HnswScalarQuantizedVectorsFormat(+ new Lucene104HnswScalarQuantizedVectorsFormat(Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH));}diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.javaindex 0952f269686..108f8bcadc6 100644--- a/lucene/core/src/java/module-info.java+++ b/lucene/core/src/java/module-info.java@@ -84,8 +84,6 @@ module org.apache.lucene.core {org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;provides org.apache.lucene.codecs.KnnVectorsFormat withorg.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat,- org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat,- org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat,org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.javaindex 4907ad67cd4..54857395c4b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java@@ -21,7 +21,6 @@* storage formats or scoring without significant changes to the HNSW code. Some examples for* scoring include {@link org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer} and {@link* org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer}. Some examples for storing include {@link- * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat} and {@link- * org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat}.+ * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat}.*/package org.apache.lucene.codecs.hnsw;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.javaindex 5f45c8cabd8..a2072a29788 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java@@ -39,6 +39,13 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {this.nonQuantizedDelegate = nonQuantizedDelegate;}+ static void checkDimensions(int queryLen, int fieldLen) {+ if (queryLen != fieldLen) {+ throw new IllegalArgumentException(+ "vector query dimension: " + queryLen + " differs from field dimension: " + fieldLen);+ }+ }+@Overridepublic RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)@@ -55,6 +62,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)throws IOException {if (vectorValues instanceof QuantizedByteVectorValues qv) {+ checkDimensions(target.length, qv.dimension());OptimizedScalarQuantizer quantizer = qv.getQuantizer();byte[] targetQuantized =new bytediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.javaindex 9acb43c64cf..44e1ef092c1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java@@ -130,6 +130,15 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {*/SEVEN_BIT(2, (byte) 7, 1);+ public static ScalarEncoding fromNumBits(int bits) {+ for (ScalarEncoding encoding : values()) {+ if (encoding.bits == bits) {+ return encoding;+ }+ }+ throw new IllegalArgumentException("No encoding for " + bits + " bits");+ }+/** The number used to identify this encoding on the wire, rather than relying on ordinal. */private final int wireNumber;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 0251224998b..8012d6095c2 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -53,9 +53,12 @@ import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector;import org.apache.lucene.util.hnsw.RandomVectorScorer;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;+import org.apache.lucene.util.quantization.QuantizedVectorsReader;+import org.apache.lucene.util.quantization.ScalarQuantizer;/** Reader for scalar quantized vectors in the Lucene 10.4 format. */-class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {+class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader+ implements QuantizedVectorsReader {private static final long SHALLOW_SIZE =RamUsageEstimator.shallowSizeOfInstance(Lucene104ScalarQuantizedVectorsReader.class);@@ -339,6 +342,64 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {return FieldEntry.create(input, vectorEncoding, info.getVectorSimilarityFunction());}+ @Override+ public org.apache.lucene.util.quantization.QuantizedByteVectorValues getQuantizedVectorValues(+ String field) throws IOException {+ FieldEntry fi = fields.get(field);+ if (fi == null) {+ return null;+ }+ if (fi.vectorEncoding != VectorEncoding.FLOAT32) {+ throw new IllegalArgumentException(+ "field=\""+ + field+ + "\" is encoded as: "+ + fi.vectorEncoding+ + " expected: "+ + VectorEncoding.FLOAT32);+ }+ var qv =+ OffHeapScalarQuantizedVectorValues.load(+ fi.ordToDocDISIReaderConfiguration,+ fi.dimension,+ fi.size,+ new OptimizedScalarQuantizer(fi.similarityFunction),+ fi.scalarEncoding,+ fi.similarityFunction,+ vectorScorer,+ fi.centroid,+ fi.centroidDP,+ fi.vectorDataOffset,+ fi.vectorDataLength,+ quantizedVectorData);+ return new org.apache.lucene.util.quantization.QuantizedByteVectorValues() {+ @Override+ public float getScoreCorrectionConstant(int ord) throws IOException {+ return 0;+ }++ @Override+ public byte[] vectorValue(int ord) throws IOException {+ return qv.vectorValue(ord);+ }++ @Override+ public int dimension() {+ return qv.dimension();+ }++ @Override+ public int size() {+ return qv.size();+ }+ };+ }++ @Override+ public ScalarQuantizer getQuantizationState(String fieldName) {+ return null;+ }+private record FieldEntry(VectorSimilarityFunction similarityFunction,VectorEncoding vectorEncoding,@@ -444,6 +505,11 @@ class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader {return quantizedVectorValues.scorer(query);}+ @Override+ public VectorScorer rescorer(float[] target) throws IOException {+ return rawVectorValues.rescorer(target);+ }+QuantizedByteVectorValues getQuantizedVectorValues() throws IOException {return quantizedVectorValues;}diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormatindex fde541c2ac0..c614014dec6 100644--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat@@ -14,8 +14,6 @@# limitations under the License.org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat-org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat-org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormatorg.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormatorg.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormatorg.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormatdiff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.javaindex cd50705ac73..7b05a018afa 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java@@ -30,7 +30,7 @@ import java.util.List;import java.util.Set;import java.util.concurrent.CountDownLatch;import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -85,7 +85,7 @@ public class TestKnnGraph extends LuceneTestCase {codec =TestUtil.alwaysKnnVectorsFormat(quantized- ? new Lucene99HnswScalarQuantizedVectorsFormat(+ ? new Lucene104HnswScalarQuantizedVectorsFormat(M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH): new Lucene99HnswVectorsFormat(M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH));diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.javaindex b849bd2a8fc..1c63ef4bf15 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java@@ -22,7 +22,8 @@ import java.util.List;import java.util.Objects;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;@@ -50,18 +51,13 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTeprivate static final int VECTOR_DIMENSION = 8;KnnVectorsFormat format;- Float confidenceInterval;int bits;@Before@Overridepublic void setUp() throws Exception {super.setUp();- bits = random().nextBoolean() ? 4 : 7;- confidenceInterval = random().nextBoolean() ? random().nextFloat(0.90f, 1.0f) : null;- if (random().nextBoolean()) {- confidenceInterval = 0f;- }+ bits = random().nextBoolean() ? 4 : 8;format = getKnnFormat(bits);savedCodec = Codec.getDefault();Codec.setDefault(getCodec());@@ -77,14 +73,12 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTereturn TestUtil.alwaysKnnVectorsFormat(format);}- private final KnnVectorsFormat getKnnFormat(int bits) {- return new Lucene99HnswScalarQuantizedVectorsFormat(+ private KnnVectorsFormat getKnnFormat(int bits) {+ return new Lucene104HnswScalarQuantizedVectorsFormat(+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.fromNumBits(bits),Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,1,- bits,- bits == 4 ? random().nextBoolean() : false,- confidenceInterval,null);}@@ -131,11 +125,11 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTew.flush();}- // index some 7 bit quantized vectors+ // index some 8 bit quantized vectorstry (IndexWriter w =new IndexWriter(dir,- newIndexWriterConfig().setCodec(TestUtil.alwaysKnnVectorsFormat(getKnnFormat(7))))) {+ newIndexWriterConfig().setCodec(TestUtil.alwaysKnnVectorsFormat(getKnnFormat(8))))) {for (int j = 0; j < numSegments; j++) {for (int i = 0; i < numVectors; i++) {Document doc = new Document();diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.javaindex e7dea987816..0e4db8679e7 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java@@ -25,7 +25,7 @@ import java.util.Map;import java.util.Random;import java.util.Set;import java.util.stream.Collectors;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.IntField;@@ -66,7 +66,7 @@ public class TestRescoreTopNQuery extends LuceneTestCase {// Set up the IndexWriterConfig to use quantized vector storageconfig = new IndexWriterConfig();config.setCodec(- TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat()));+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));}@Testdiff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.javaindex 1f6c5d525e4..653c519729a 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java@@ -25,7 +25,7 @@ import java.util.List;import java.util.concurrent.Executor;import java.util.concurrent.ForkJoinPool;import java.util.concurrent.ForkJoinWorkerThread;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.document.StoredField;@@ -63,7 +63,7 @@ public class TestBpVectorReorderer extends LuceneTestCase {private void createQuantizedIndex(Directory dir, List<float[]> vectors) throws IOException {IndexWriterConfig cfg = new IndexWriterConfig();cfg.setCodec(- TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat(8, 32)));+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat(8, 32)));try (IndexWriter writer = new IndexWriter(dir, cfg)) {int i = 0;for (float[] vector : vectors) {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.javaindex 6539423e702..420d6d0164e 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java@@ -33,10 +33,11 @@ import org.apache.lucene.codecs.PointsReader;import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;-import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.memory.DirectPostingsFormat;import org.apache.lucene.codecs.memory.FSTPostingsFormat;@@ -272,32 +273,24 @@ public class RandomCodec extends AssertingCodec {TestUtil.nextInt(random, 10, 50),concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,concurrentKnnMerging ? ForkJoinPool.commonPool() : null),- new Lucene99HnswScalarQuantizedVectorsFormat(+ new Lucene104HnswScalarQuantizedVectorsFormat(+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT,+ TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),+ new Lucene104HnswScalarQuantizedVectorsFormat(+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,+ TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),+ new Lucene104HnswScalarQuantizedVectorsFormat(+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.PACKED_NIBBLE,TestUtil.nextInt(random, 5, 50),TestUtil.nextInt(random, 10, 50),concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,- 7,- false,- randomConfidenceInterval(random),concurrentKnnMerging ? ForkJoinPool.commonPool() : null),- // TODO: also test 4-bit quantization, but this must somehow be restricted to even-length- // fields- /*- * new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),- * TestUtil.nextInt(random, 10, 50),- * 1,- * 4,- * random.nextBoolean(),- * randomConfidenceInterval(random),- * null),- * new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),- * TestUtil.nextInt(random, 10, 50),- * TestUtil.nextInt(random, 2, 8),- * 4,- * random.nextBoolean(),- * randomConfidenceInterval(random),- * ForkJoinPool.commonPool()),- */new AssertingKnnVectorsFormat());Collections.shuffle(formats, random);@@ -316,18 +309,6 @@ public class RandomCodec extends AssertingCodec {}}- private final Float randomConfidenceInterval(Random random) {- switch (random.nextInt(3)) {- default:- case 0:- return null;- case 1:- return 0f;- case 2:- return random.nextFloat(0.9f, 1f);- }- }-public RandomCodec(Random random) {this(random, Collections.<String>emptySet());}