Apache Solr open-source search software
SOLR-18267: Add scalar quantized flat vector index support with no HNSW (#4698)
Adam Quigley64feac04c7d8
diff --git a/changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml b/changelog/unreleased/SOLR-18267-scalar-quantized-flat.ymlnew file mode 100644index 00000000000..b16cf1c46fd--- /dev/null+++ b/changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml@@ -0,0 +1,8 @@+title: Support knnAlgorithm="flat" for ScalarQuantizedDenseVectorField to store scalar-quantized vectors+ without building an HNSW graph.+type: added+authors:+- name: Adam Quigley+links:+- name: SOLR-18267+ url: https://issues.apache.org/jira/browse/SOLR-18267diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.javaindex 9ac1aae556b..e06dfa6127b 100644--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java@@ -490,6 +490,16 @@ public class DenseVectorField extends FloatPointField {SolrException.ErrorCode.BAD_REQUEST, "Vector encoding not supported for function queries.");}+ /** Throws if this field type does not support the KNN query parsers. */+ public void checkKnnQueryParsersSupported() throws SolrException {+ if (FLAT_ALGORITHM.equals(knnAlgorithm)) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". "+ + "Use vectorSimilarity() function queries instead.");+ }+ }+public Query getKnnVectorQuery(String fieldName,String vectorToSearch,@@ -500,12 +510,7 @@ public class DenseVectorField extends FloatPointField {EarlyTerminationParams earlyTermination,Integer filteredSearchThreshold) {- if (FLAT_ALGORITHM.equals(knnAlgorithm)) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST,- "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". "- + "Use vectorSimilarity() function queries instead.");- }+ checkKnnQueryParsersSupported();DenseVectorParser vectorBuilder =getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY);diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.javaindex 66f0387505d..fb5047ae4c2 100644--- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java@@ -21,6 +21,7 @@ import static java.util.Optional.ofNullable;import java.util.Map;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;@@ -95,19 +96,29 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {}super.init(schema, args);+ }- if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {+ @Override+ public void checkKnnQueryParsersSupported() throws SolrException {+ if (FLAT_ALGORITHM.equals(getKnnAlgorithm())+ && VectorEncoding.BYTE.equals(getVectorEncoding())) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,- "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField");+ "KNN vector queries are not supported for ScalarQuantizedDenseVectorField using "+ + "knnAlgorithm=\"flat\" with vectorEncoding=\"BYTE\". "+ + "Use vectorSimilarity() function queries instead.");}}@Overridepublic KnnVectorsFormat buildKnnVectorsFormat() {ScalarEncoding encoding = ScalarEncoding.fromNumBits(getBits());- return new Lucene104HnswScalarQuantizedVectorsFormat(- encoding, getHnswM(), getHnswEfConstruction());+ if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {+ return new Lucene104ScalarQuantizedVectorsFormat(encoding);+ } else {+ return new Lucene104HnswScalarQuantizedVectorsFormat(+ encoding, getHnswM(), getHnswEfConstruction());+ }}@Overridediff --git a/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.javaindex eb533582769..bd1b6b43835 100644--- a/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java+++ b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java@@ -120,12 +120,7 @@ public class KnnQParser extends AbstractVectorQParserBase {final SchemaField schemaField = req.getSchema().getField(getFieldName());final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);- if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST,- "The {!knn} query parser is not supported for fields using knnAlgorithm=\"flat\". "- + "Use vectorSimilarity() function queries instead.");- }+ denseVectorType.checkKnnQueryParsersSupported();final String vectorToSearch = getVectorToSearch();final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);diff --git a/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.javaindex 17279d2d1a5..e4c72366e9e 100644--- a/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java+++ b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java@@ -47,12 +47,7 @@ public class VectorSimilarityQParser extends AbstractVectorQParserBase {final SchemaField schemaField = req.getSchema().getField(fieldName);final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);- if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST,- "The {!vectorSimilarity} query parser is not supported for fields using knnAlgorithm=\"flat\". "- + "Use vectorSimilarity() function queries instead.");- }+ denseVectorType.checkKnnQueryParsersSupported();final String vectorToSearch = getVectorToSearch();final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE);diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xmlsimilarity index 69%rename from solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xmlrename to solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xmlindex 365bddd03e7..2da0dab09c6 100644--- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml@@ -16,14 +16,16 @@limitations under the License.-->-<!-- Test schema file for DenseVectorField -->+<!-- Test schema file for ScalarQuantizedDenseVectorField with flat algorithm -->-<schema name="bad-schema-densevector-flat-quantized" version="1.7">+<schema name="schema-densevector-flat-scalarQuantized" version="1.7"><fieldType name="string" class="solr.StrField" multiValued="true"/><fieldType name="knn_vector_flat_sq" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>+ <fieldType name="knn_vector_flat_sq_byte" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat" vectorEncoding="BYTE"/><field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>- <field name="vector" type="knn_vector_flat_sq" indexed="true" stored="true"/>+ <field name="vector_sq_flat" type="knn_vector_flat_sq" indexed="true" stored="true"/>+ <field name="vector_sq_flat_byte" type="knn_vector_flat_sq_byte" indexed="true" stored="true"/><uniqueKey>id</uniqueKey></schema>diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.javaindex cb1f24b88d5..a864b022283 100644--- a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java+++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java@@ -18,7 +18,12 @@ package org.apache.solr.schema;import static org.hamcrest.core.Is.is;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;import org.apache.solr.core.AbstractBadConfigTestBase;import org.junit.Test;@@ -129,10 +134,227 @@ public class ScalarQuantizedDenseVectorFieldTest extends AbstractBadConfigTestBa}@Test- public void fieldDefinition_flatAlgorithm_shouldThrowException() throws Exception {- assertConfigs(- "solrconfig-basic.xml",- "bad-schema-densevector-flat-scalarQuantized.xml",- "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField");+ public void fieldDefinition_flatAlgorithm_shouldLoadSchemaField() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");+ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vector = schema.getField("vector_sq_flat");+ assertNotNull(vector);++ ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType();+ assertThat(type.getKnnAlgorithm(), is("flat"));+ assertThat(type.getDimension(), is(4));+ assertThat(type.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));+ assertThat(type.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));++ assertTrue(vector.indexed());+ assertTrue(vector.stored());+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_buildKnnVectorsFormat_shouldReturnScalarQuantizedFormat()+ throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");+ IndexSchema schema = h.getCore().getLatestSchema();++ SchemaField vector = schema.getField("vector_sq_flat");+ ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType();++ assertThat(+ type.buildKnnVectorsFormat() instanceof Lucene104ScalarQuantizedVectorsFormat, is(true));+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_vectorSimilarityFunction_shouldReturnResults() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ addVectorDoc("0", "vector_sq_flat", 2.0f, 3.0f, 4.0f, 5.0f);+ addVectorDoc("1", "vector_sq_flat", 100.0f, 200.0f, 50.0f, 25.0f);+ addVectorDoc("2", "vector_sq_flat", 1.0f, 2.0f, 3.0f, 4.0f);+ addVectorDoc("3", "vector_sq_flat", -1.0f, -2.0f, -3.0f, -4.0f);++ assertU(commit());++ // {!func}vectorSimilarity returns all results if not limited+ assertJQ(+ req(+ "q", "{!func}vectorSimilarity(vector_sq_flat,[1.0, 2.0, 3.0, 4.0])",+ "fl", "id,score"),+ "/response/numFound==4",+ "/response/docs/[0]/id=='2'",+ "/response/docs/[1]/id=='0'",+ "/response/docs/[2]/id=='1'",+ "/response/docs/[3]/id=='3'");+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_knnQuery_shouldReturnResults() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ addVectorDoc("0", "vector_sq_flat", 2.0f, 3.0f, 4.0f, 5.0f);+ addVectorDoc("1", "vector_sq_flat", 100.0f, 200.0f, 50.0f, 25.0f);+ addVectorDoc("2", "vector_sq_flat", 1.0f, 2.0f, 3.0f, 4.0f);+ addVectorDoc("3", "vector_sq_flat", -1.0f, -2.0f, -3.0f, -4.0f);++ assertU(commit());++ assertJQ(+ req(+ "q", "{!knn f=vector_sq_flat topK=3}[1.0, 2.0, 3.0, 4.0]",+ "fl", "id,score"),+ "/response/numFound==3",+ "/response/docs/[0]/id=='2'",+ "/response/docs/[1]/id=='0'",+ "/response/docs/[2]/id=='1'");+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_knnQuery_preFilter_shouldReturnFilteredResults() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ addVectorDoc("0", "vector_sq_flat", 2.0f, 3.0f, 4.0f, 5.0f);+ addVectorDoc("1", "vector_sq_flat", 100.0f, 200.0f, 50.0f, 25.0f);+ addVectorDoc("2", "vector_sq_flat", 1.0f, 2.0f, 3.0f, 4.0f);+ addVectorDoc("3", "vector_sq_flat", -1.0f, -2.0f, -3.0f, -4.0f);++ assertU(commit());++ // id 0 is the second-best match but is excluded by the preFilter+ assertJQ(+ req(+ "q", "{!knn f=vector_sq_flat topK=3 preFilter='id:(1 2 3)'}[1.0, 2.0, 3.0, 4.0]",+ "fl", "id,score"),+ "/response/numFound==3",+ "/response/docs/[0]/id=='2'",+ "/response/docs/[1]/id=='1'",+ "/response/docs/[2]/id=='3'");+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_vectorSimilarityQParser_shouldReturnResults() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ addVectorDoc("0", "vector_sq_flat", 2.0f, 3.0f, 4.0f, 5.0f);+ addVectorDoc("1", "vector_sq_flat", 100.0f, 200.0f, 50.0f, 25.0f);+ addVectorDoc("2", "vector_sq_flat", 1.0f, 2.0f, 3.0f, 4.0f);+ addVectorDoc("3", "vector_sq_flat", -1.0f, -2.0f, -3.0f, -4.0f);++ assertU(commit());++ // id 3 is excluded by minReturn+ assertJQ(+ req(+ "q", "{!vectorSimilarity f=vector_sq_flat minReturn=0.5}[1.0, 2.0, 3.0, 4.0]",+ "fl", "id,score"),+ "/response/numFound==3",+ "/response/docs/[0]/id=='2'",+ "/response/docs/[1]/id=='0'",+ "/response/docs/[2]/id=='1'");+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_byteEncoding_knnQuery_shouldThrowException() throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ assertQEx(+ "Running {!knn} on a flat scalar quantized BYTE vector field should raise an Exception",+ "vectorEncoding=\"BYTE\"",+ req("q", "{!knn f=vector_sq_flat_byte topK=2}[1, 2, 3, 4]", "fl", "id"),+ SolrException.ErrorCode.BAD_REQUEST);+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_byteEncoding_vectorSimilarityQParser_shouldThrowException()+ throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ assertQEx(+ "Running {!vectorSimilarity} on a flat scalar quantized BYTE vector field should raise an Exception",+ "vectorEncoding=\"BYTE\"",+ req(+ "q", "{!vectorSimilarity f=vector_sq_flat_byte minReturn=0.99}[1, 2, 3, 4]",+ "fl", "id"),+ SolrException.ErrorCode.BAD_REQUEST);+ } finally {+ deleteCore();+ }+ }++ @Test+ public void flatAlgorithm_byteEncoding_vectorSimilarityFunction_shouldReturnResults()+ throws Exception {+ try {+ initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");++ addVectorDoc("0", "vector_sq_flat_byte", 5, 6, 7, 8);+ addVectorDoc("1", "vector_sq_flat_byte", 1, 2, 3, 4);+ addVectorDoc("2", "vector_sq_flat_byte", -1, -2, -3, -4);++ assertU(commit());++ // {!func}vectorSimilarity returns all results if not limited+ assertJQ(+ req(+ "q", "{!func}vectorSimilarity(vector_sq_flat_byte,[1, 2, 3, 4])",+ "fl", "id,score"),+ "/response/numFound==3",+ "/response/docs/[0]/id=='1'",+ "/response/docs/[0]/score==1.0",+ "/response/docs/[1]/id=='0'",+ "/response/docs/[2]/id=='2'");+ } finally {+ deleteCore();+ }+ }++ private void addVectorDoc(String id, String vectorField, float... v) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", id);+ List<Float> vector = new ArrayList<>(v.length);+ for (float value : v) {+ vector.add(value);+ }+ doc.addField(vectorField, vector);+ assertU(adoc(doc));+ }++ private void addVectorDoc(String id, String vectorField, int... v) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", id);+ List<Integer> vector = new ArrayList<>(v.length);+ for (int value : v) {+ vector.add(value);+ }+ doc.addField(vectorField, vector);+ assertU(adoc(doc));}}diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adocindex 879ec439124..a10ee670ee3 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc@@ -128,7 +128,7 @@ Here's how `DenseVectorField` can be configured with the advanced hyperparameter+Accepted values: `hnsw`, `flat`, `cagra_hnsw` (requires GPU acceleration setup).+-The `flat` algorithm stores vectors without building an HNSW graph. This avoids the indexing overhead of graph construction, but does not support the `{!knn}` query parser. Use `vectorSimilarity()` function queries to score and rank documents by vector similarity. See <<Flat Vector Index>> for details.+The `flat` algorithm stores vectors without building an HNSW graph, avoiding the indexing overhead of graph construction. Supported query parsers differ by field type. See <<Flat Vector Index>> for details.+Please note that the `knnAlgorithm` accepted values may change in future releases.@@ -300,8 +300,9 @@ with 8 bits we can store up to 256 discrete values, so a float dimension with va[0.0, 0.0039) => 0, [0.0039, 0.0078) => 1 ... etc-As a specific type of DenseVectorField, this field type supports all the same configurable properties outlined above as well-as some additional ones.+As a specific type of DenseVectorField, this field type supports `knnAlgorithm` (`hnsw` or `flat`) and other configurable properties outlined above, as well as some additional ones.++NOTE: Scalar quantization applies to `FLOAT32` vectors only. `vectorEncoding="BYTE"` vectors are stored unquantized, and with `knnAlgorithm="flat"` they do not support the KNN query parsers. Use `vectorSimilarity()` function queries instead.Here is how a ScalarQuantizedDenseVectorField can be defined in the schema:@@ -348,6 +349,8 @@ BinaryQuantizedDenseVectorField accepts the same parameters as `DenseVectorField`similarityFunction`. Bit quantization uses its own distance calculation and so does not require nor use the `similarityFunction`param.+NOTE: `knnAlgorithm="flat"` is not supported for `BinaryQuantizedDenseVectorField`.+[[query-hnsw-fields]]== Querying Vectors in Navigable Small-world Graphs@@ -873,12 +876,28 @@ The final ranked list of results will have the first pass score(main query `q`)Setting `knnAlgorithm="flat"` stores vectors without building an HNSW graph, avoiding the indexing cost of graph construction.-Flat fields do not support the `{!knn}`, `{!knn_text_to_vector}`, or `{!vectorSimilarity}` query parsers.+The flat algorithm is supported by both `DenseVectorField` and `ScalarQuantizedDenseVectorField`. When used with `ScalarQuantizedDenseVectorField`, `FLOAT32` vectors are still quantized at index time.++[source,xml]+----+<!-- Flat vector index (no quantization) -->+<fieldType name="knn_flat" class="solr.DenseVectorField"+ vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>++<!-- Flat vector index with scalar quantization -->+<fieldType name="knn_sq_flat" class="solr.ScalarQuantizedDenseVectorField"+ vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>+----++For `DenseVectorField`, flat fields do not support the `{!knn}`, `{!knn_text_to_vector}`, or `{!vectorSimilarity}` query parsers.Use `vectorSimilarity()` function queries to score and rank by similarity:[source]q={!func}vectorSimilarity(vector,[1.0, 2.0, 3.0, 4.0])&fl=id,score+For `ScalarQuantizedDenseVectorField` with `FLOAT32` vectors, flat fields also support the `{!knn}`, `{!knn_text_to_vector}`, and `{!vectorSimilarity}` query parsers, which score against the quantized vectors.+Note that `vectorSimilarity()` function queries score against the raw (unquantized) float vectors, so the two paths can return slightly different scores for the same field.+== Indexing Multi-Vectors for Late Interaction