Apache Solr open-source search software
SOLR-17927: Fix naming in vector search support (#3830)
Chaitali Rajhans0f5b2c440006
diff --git a/changelog/unreleased/solr_17927_new.yml b/changelog/unreleased/solr_17927_new.ymlnew file mode 100644index 00000000000..862f05037c4--- /dev/null+++ b/changelog/unreleased/solr_17927_new.yml@@ -0,0 +1,8 @@+issue: SOLR-17927+title: Rename vector search parameters and language model modules+type: changed+authors:+ - name: Ishan Chattopadhyaya+ - name: Alessandro Benedetti+ - name: Ilaria Petreti+ - name: Chaitali Rajhans\ No newline at end of filediff --git a/gradlew.bat b/gradlew.batindex ff65d8f6012..699ee221a92 100644--- a/gradlew.bat+++ b/gradlew.bat@@ -1,126 +1,126 @@-@rem-@rem Copyright 2015 the original author or authors.-@rem-@rem Licensed under the Apache License, Version 2.0 (the "License");-@rem you may not use this file except in compliance with the License.-@rem You may obtain a copy of the License at-@rem-@rem https://www.apache.org/licenses/LICENSE-2.0-@rem-@rem Unless required by applicable law or agreed to in writing, software-@rem distributed under the License is distributed on an "AS IS" BASIS,-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-@rem See the License for the specific language governing permissions and-@rem limitations under the License.-@rem-@rem SPDX-License-Identifier: Apache-2.0-@rem--@if "%DEBUG%"=="" @echo off-@rem ##########################################################################-@rem-@rem Gradle startup script for Windows-@rem-@rem ##########################################################################--@rem Set local scope for the variables with windows NT shell-if "%OS%"=="Windows_NT" setlocal--set DIRNAME=%~dp0-if "%DIRNAME%"=="" set DIRNAME=.-@rem This is normally unused-set APP_BASE_NAME=%~n0-set APP_HOME=%DIRNAME%--@rem Resolve any "." and ".." in APP_HOME to make it shorter.-for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi--@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"--@rem LUCENE-9471: workaround for gradle leaving junk temp. files behind.-SET GRADLE_TEMPDIR=%DIRNAME%\.gradle\tmp-IF NOT EXIST "%GRADLE_TEMPDIR%" MKDIR "%GRADLE_TEMPDIR%"-SET DEFAULT_JVM_OPTS=%DEFAULT_JVM_OPTS% "-Djava.io.tmpdir=%GRADLE_TEMPDIR%"--@rem Find java.exe-if defined JAVA_HOME goto findJavaFromJavaHome--set JAVA_EXE=java.exe-%JAVA_EXE% -version >NUL 2>&1-if %ERRORLEVEL% equ 0 goto execute--echo. 1>&2-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2-echo. 1>&2-echo Please set the JAVA_HOME variable in your environment to match the 1>&2-echo location of your Java installation. 1>&2--goto fail--:findJavaFromJavaHome-set JAVA_HOME=%JAVA_HOME:"=%-set JAVA_EXE=%JAVA_HOME%/bin/java.exe--if exist "%JAVA_EXE%" goto execute--echo. 1>&2-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2-echo. 1>&2-echo Please set the JAVA_HOME variable in your environment to match the 1>&2-echo location of your Java installation. 1>&2--goto fail--:execute-@rem Setup the command line--@rem LUCENE-9266: verify and download the gradle wrapper jar if we don't have one.-set GRADLE_WRAPPER_JAR=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar-IF NOT EXIST "%GRADLE_WRAPPER_JAR%" (- "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "%GRADLE_WRAPPER_JAR%"- IF %ERRORLEVEL% EQU 1 goto failWithJvmMessage- IF %ERRORLEVEL% NEQ 0 goto fail-)--@rem Setup the command line-set CLASSPATH=%GRADLE_WRAPPER_JAR%--@rem START OF LUCENE CUSTOMIZATION-@rem Generate gradle.properties if they don't exist-IF NOT EXIST "%APP_HOME%\gradle.properties" (- @rem local expansion is needed to check ERRORLEVEL inside control blocks.- setlocal enableDelayedExpansion- "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"- IF %ERRORLEVEL% NEQ 0 goto fail- endlocal-)-@rem END OF LUCENE CUSTOMIZATION--@rem Prevent jgit from forking/searching git.exe-SET GIT_CONFIG_NOSYSTEM=1--@rem Execute Gradle-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*--:end-@rem End local scope for the variables with windows NT shell-if %ERRORLEVEL% equ 0 goto mainEnd-goto fail--:failWithJvmMessage-@rem https://github.com/apache/lucene/pull/819-echo Error: Something went wrong. Make sure you're using Java version between 21 and 23.--:fail-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of-rem the _cmd.exe /c_ return code!-set EXIT_CODE=%ERRORLEVEL%-if %EXIT_CODE% equ 0 set EXIT_CODE=1-if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%-exit /b %EXIT_CODE%--:mainEnd-if "%OS%"=="Windows_NT" endlocal--:omega+@rem+@rem Copyright 2015 the original author or authors.+@rem+@rem Licensed under the Apache License, Version 2.0 (the "License");+@rem you may not use this file except in compliance with the License.+@rem You may obtain a copy of the License at+@rem+@rem https://www.apache.org/licenses/LICENSE-2.0+@rem+@rem Unless required by applicable law or agreed to in writing, software+@rem distributed under the License is distributed on an "AS IS" BASIS,+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+@rem See the License for the specific language governing permissions and+@rem limitations under the License.+@rem+@rem SPDX-License-Identifier: Apache-2.0+@rem++@if "%DEBUG%"=="" @echo off+@rem ##########################################################################+@rem+@rem Gradle startup script for Windows+@rem+@rem ##########################################################################++@rem Set local scope for the variables with windows NT shell+if "%OS%"=="Windows_NT" setlocal++set DIRNAME=%~dp0+if "%DIRNAME%"=="" set DIRNAME=.+@rem This is normally unused+set APP_BASE_NAME=%~n0+set APP_HOME=%DIRNAME%++@rem Resolve any "." and ".." in APP_HOME to make it shorter.+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi++@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"++@rem LUCENE-9471: workaround for gradle leaving junk temp. files behind.+SET GRADLE_TEMPDIR=%DIRNAME%\.gradle\tmp+IF NOT EXIST "%GRADLE_TEMPDIR%" MKDIR "%GRADLE_TEMPDIR%"+SET DEFAULT_JVM_OPTS=%DEFAULT_JVM_OPTS% "-Djava.io.tmpdir=%GRADLE_TEMPDIR%"++@rem Find java.exe+if defined JAVA_HOME goto findJavaFromJavaHome++set JAVA_EXE=java.exe+%JAVA_EXE% -version >NUL 2>&1+if %ERRORLEVEL% equ 0 goto execute++echo. 1>&2+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2+echo. 1>&2+echo Please set the JAVA_HOME variable in your environment to match the 1>&2+echo location of your Java installation. 1>&2++goto fail++:findJavaFromJavaHome+set JAVA_HOME=%JAVA_HOME:"=%+set JAVA_EXE=%JAVA_HOME%/bin/java.exe++if exist "%JAVA_EXE%" goto execute++echo. 1>&2+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2+echo. 1>&2+echo Please set the JAVA_HOME variable in your environment to match the 1>&2+echo location of your Java installation. 1>&2++goto fail++:execute+@rem Setup the command line++@rem LUCENE-9266: verify and download the gradle wrapper jar if we don't have one.+set GRADLE_WRAPPER_JAR=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar+IF NOT EXIST "%GRADLE_WRAPPER_JAR%" (+ "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java" "%GRADLE_WRAPPER_JAR%"+ IF %ERRORLEVEL% EQU 1 goto failWithJvmMessage+ IF %ERRORLEVEL% NEQ 0 goto fail+)++@rem Setup the command line+set CLASSPATH=%GRADLE_WRAPPER_JAR%++@rem START OF LUCENE CUSTOMIZATION+@rem Generate gradle.properties if they don't exist+IF NOT EXIST "%APP_HOME%\gradle.properties" (+ @rem local expansion is needed to check ERRORLEVEL inside control blocks.+ setlocal enableDelayedExpansion+ "%JAVA_EXE%" %JAVA_OPTS% "%APP_HOME%/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"+ IF %ERRORLEVEL% NEQ 0 goto fail+ endlocal+)+@rem END OF LUCENE CUSTOMIZATION++@rem Prevent jgit from forking/searching git.exe+SET GIT_CONFIG_NOSYSTEM=1++@rem Execute Gradle+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*++:end+@rem End local scope for the variables with windows NT shell+if %ERRORLEVEL% equ 0 goto mainEnd+goto fail++:failWithJvmMessage+@rem https://github.com/apache/lucene/pull/819+echo Error: Something went wrong. Make sure you're using Java version between 21 and 23.++:fail+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of+rem the _cmd.exe /c_ return code!+set EXIT_CODE=%ERRORLEVEL%+if %EXIT_CODE% equ 0 set EXIT_CODE=1+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%+exit /b %EXIT_CODE%++:mainEnd+if "%OS%"=="Windows_NT" endlocal++:omegadiff --git a/settings.gradle b/settings.gradleindex 7b635cbbeb9..08cee48a39a 100644--- a/settings.gradle+++ b/settings.gradle@@ -52,7 +52,7 @@ include "solr:modules:extraction"include "solr:modules:gcs-repository"include "solr:modules:jwt-auth"include "solr:modules:langid"-include "solr:modules:llm"+include "solr:modules:language-models"include "solr:modules:ltr"include "solr:modules:s3-repository"include "solr:modules:scripting"diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/BinaryQuantizedDenseVectorField.javaindex 4293e9a6d8b..8f172b27373 100644--- a/solr/core/src/java/org/apache/solr/schema/BinaryQuantizedDenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/BinaryQuantizedDenseVectorField.java@@ -23,6 +23,6 @@ public class BinaryQuantizedDenseVectorField extends DenseVectorField {@Overridepublic KnnVectorsFormat buildKnnVectorsFormat() {- return new Lucene102HnswBinaryQuantizedVectorsFormat(getHnswMaxConn(), getHnswBeamWidth());+ return new Lucene102HnswBinaryQuantizedVectorsFormat(getHnswM(), getHnswEfConstruction());}}diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.javaindex f1831c1ad9c..21c8d0d789c 100644--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java@@ -48,7 +48,7 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.hnsw.HnswGraph;import org.apache.solr.common.SolrException;import org.apache.solr.search.QParser;-import org.apache.solr.search.neural.KnnQParser.EarlyTerminationParams;+import org.apache.solr.search.vector.KnnQParser.EarlyTerminationParams;import org.apache.solr.uninverting.UninvertingReader;import org.apache.solr.util.vector.ByteDenseVectorParser;import org.apache.solr.util.vector.DenseVectorParser;@@ -75,6 +75,8 @@ public class DenseVectorField extends FloatPointField {static final String KNN_ALGORITHM = "knnAlgorithm";static final String HNSW_MAX_CONNECTIONS = "hnswMaxConnections";static final String HNSW_BEAM_WIDTH = "hnswBeamWidth";+ static final String HNSW_M = "hnswM";+ static final String HNSW_EF_CONSTRUCTION = "hnswEfConstruction";static final String VECTOR_ENCODING = "vectorEncoding";static final VectorEncoding DEFAULT_VECTOR_ENCODING = VectorEncoding.FLOAT32;static final String KNN_SIMILARITY_FUNCTION = "similarityFunction";@@ -101,13 +103,13 @@ public class DenseVectorField extends FloatPointField {* This parameter is coupled with the hnsw algorithm. Controls how many of the nearest neighbor* candidates are connected to the new node. See {@link HnswGraph} for more details.*/- private int hnswMaxConn;+ private int hnswM;/*** This parameter is coupled with the hnsw algorithm. The number of candidate neighbors to track* while searching the graph for each newly inserted node. See {@link HnswGraph} for details.*/- private int hnswBeamWidth;+ private int hnswEfConstruction;/*** Encoding for vector value representation. The possible values are FLOAT32 or BYTE. The default@@ -142,8 +144,24 @@ public class DenseVectorField extends FloatPointField {this.vectorEncoding = vectorEncoding;}+ private String getFieldTypeName(IndexSchema schema) {+ return schema != null ? schema.getSchemaName() : "unknown";+ }+@Overridepublic void init(IndexSchema schema, Map<String, String> args) {++ if (args.containsKey("hnswMaxConnections")) {+ log.warn(+ "Deprecated parameter 'hnswMaxConnections' detected in fieldType '{}'. Use 'hnswM' instead.",+ getFieldTypeName(schema));+ }+ if (args.containsKey("hnswBeamWidth")) {+ log.warn(+ "Deprecated parameter 'hnswBeamWidth' detected in fieldType '{}'. Use 'hnswEfConstruction' instead.",+ getFieldTypeName(schema));+ }+this.dimension =ofNullable(args.get(KNN_VECTOR_DIMENSION)).map(Integer::parseInt)@@ -169,12 +187,26 @@ public class DenseVectorField extends FloatPointField {.orElse(DEFAULT_VECTOR_ENCODING);args.remove(VECTOR_ENCODING);- this.hnswMaxConn =- ofNullable(args.get(HNSW_MAX_CONNECTIONS)).map(Integer::parseInt).orElse(DEFAULT_MAX_CONN);+ this.hnswM =+ ofNullable(args.get(HNSW_M))+ .map(Integer::parseInt)+ .orElseGet(+ () ->+ ofNullable(args.get(HNSW_MAX_CONNECTIONS))+ .map(Integer::parseInt)+ .orElse(DEFAULT_MAX_CONN));+ args.remove(HNSW_M);args.remove(HNSW_MAX_CONNECTIONS);- this.hnswBeamWidth =- ofNullable(args.get(HNSW_BEAM_WIDTH)).map(Integer::parseInt).orElse(DEFAULT_BEAM_WIDTH);+ this.hnswEfConstruction =+ ofNullable(args.get(HNSW_EF_CONSTRUCTION))+ .map(Integer::parseInt)+ .orElseGet(+ () ->+ ofNullable(args.get(HNSW_BEAM_WIDTH))+ .map(Integer::parseInt)+ .orElse(DEFAULT_BEAM_WIDTH));+ args.remove(HNSW_EF_CONSTRUCTION);args.remove(HNSW_BEAM_WIDTH);this.cuvsWriterThreads =@@ -231,12 +263,22 @@ public class DenseVectorField extends FloatPointField {return knnAlgorithm;}+ @Deprecatedpublic Integer getHnswMaxConn() {- return hnswMaxConn;+ return hnswM;}+ @Deprecatedpublic Integer getHnswBeamWidth() {- return hnswBeamWidth;+ return hnswEfConstruction;+ }++ public Integer getHnswM() {+ return hnswM;+ }++ public Integer getHnswEfConstruction() {+ return hnswEfConstruction;}public VectorEncoding getVectorEncoding() {@@ -434,7 +476,7 @@ public class DenseVectorField extends FloatPointField {}public KnnVectorsFormat buildKnnVectorsFormat() {- return new Lucene99HnswVectorsFormat(hnswMaxConn, hnswBeamWidth);+ return new Lucene99HnswVectorsFormat(hnswM, hnswEfConstruction);}@Overridediff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.javaindex 885d09cea17..9a635cd0b1f 100644--- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java@@ -96,8 +96,8 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {@Overridepublic KnnVectorsFormat buildKnnVectorsFormat() {return new Lucene99HnswScalarQuantizedVectorsFormat(- getHnswMaxConn(),- getHnswBeamWidth(),+ getHnswM(),+ getHnswEfConstruction(),DEFAULT_NUM_MERGE_WORKER,getBits(),useCompression(),diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.javaindex 80855aa0bdf..45409cb8982 100644--- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java@@ -28,8 +28,8 @@ import org.apache.solr.search.join.GraphQParserPlugin;import org.apache.solr.search.join.HashRangeQParserPlugin;import org.apache.solr.search.mlt.MLTContentQParserPlugin;import org.apache.solr.search.mlt.MLTQParserPlugin;-import org.apache.solr.search.neural.KnnQParserPlugin;-import org.apache.solr.search.neural.VectorSimilarityQParserPlugin;+import org.apache.solr.search.vector.KnnQParserPlugin;+import org.apache.solr.search.vector.VectorSimilarityQParserPlugin;import org.apache.solr.util.plugin.NamedListInitializedPlugin;public abstract class QParserPlugin implements NamedListInitializedPlugin {diff --git a/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java b/solr/core/src/java/org/apache/solr/search/vector/AbstractVectorQParserBase.javasimilarity index 99%rename from solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.javarename to solr/core/src/java/org/apache/solr/search/vector/AbstractVectorQParserBase.javaindex 065180d7e4a..ec5426d6d93 100644--- a/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java+++ b/solr/core/src/java/org/apache/solr/search/vector/AbstractVectorQParserBase.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import java.io.IOException;import java.util.ArrayList;diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.javasimilarity index 99%rename from solr/core/src/java/org/apache/solr/search/neural/KnnQParser.javarename to solr/core/src/java/org/apache/solr/search/vector/KnnQParser.javaindex db355e0b84e..08fa6a5fc9a 100644--- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java+++ b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import java.util.Optional;import org.apache.lucene.search.Query;diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/vector/KnnQParserPlugin.javasimilarity index 93%rename from solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.javarename to solr/core/src/java/org/apache/solr/search/vector/KnnQParserPlugin.javaindex 29e5c28e879..80f5445c017 100644--- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/vector/KnnQParserPlugin.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import org.apache.solr.common.params.SolrParams;import org.apache.solr.request.SolrQueryRequest;@@ -22,7 +22,7 @@ import org.apache.solr.search.QParser;import org.apache.solr.search.QParserPlugin;/**- * A neural query parser to run K-nearest neighbors search on Dense Vector fields. See Wiki page+ * A vector query parser to run K-nearest neighbors search on Dense Vector fields. See Wiki page* https://solr.apache.org/guide/solr/latest/query-guide/dense-vector-search.html*/public class KnnQParserPlugin extends QParserPlugin {diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.javasimilarity index 98%rename from solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.javarename to solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.javaindex e3ec2f242f7..f2d9be3738a 100644--- a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java+++ b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.search.ByteVectorSimilarityQuery;diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParserPlugin.javasimilarity index 92%rename from solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.javarename to solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParserPlugin.javaindex 4109d0df388..ec6bfd8b155 100644--- a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java+++ b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParserPlugin.java@@ -14,14 +14,14 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import org.apache.solr.common.params.SolrParams;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.search.QParser;import org.apache.solr.search.QParserPlugin;-/** A neural query parser to run min-similarity search on Dense Vector fields. */+/** A vector query parser to run min-similarity search on Dense Vector fields. */public class VectorSimilarityQParserPlugin extends QParserPlugin {public static final String NAME = "vectorSimilarity";diff --git a/solr/core/src/java/org/apache/solr/search/neural/package-info.java b/solr/core/src/java/org/apache/solr/search/vector/package-info.javasimilarity index 88%rename from solr/core/src/java/org/apache/solr/search/neural/package-info.javarename to solr/core/src/java/org/apache/solr/search/vector/package-info.javaindex f4343541338..538c98d168b 100644--- a/solr/core/src/java/org/apache/solr/search/neural/package-info.java+++ b/solr/core/src/java/org/apache/solr/search/vector/package-info.java@@ -15,5 +15,5 @@* limitations under the License.*/-/** APIs and classes for implementing Neural (Dense Retrieval) QueryParsers. */-package org.apache.solr.search.neural;+/** APIs and classes for implementing Vector (Dense Retrieval) QueryParsers. */+package org.apache.solr.search.vector;diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparameter.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparameter.xmlindex 2f35ebfc5e6..a61de8b6466 100644--- a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparameter.xml+++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-codec-hyperparameter.xml@@ -21,9 +21,9 @@<schema name="schema-densevector-codec-hyperparameter" version="1.7"><fieldType name="string" class="solr.StrField" multiValued="true"/><fieldType name="knn_vector_default" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine"/>- <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswMaxConnections="10" hnswBeamWidth="40"/>- <fieldType name="knn_vector2" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswMaxConnections="6" hnswBeamWidth="60"/>- <fieldType name="knn_vector3" class="solr.DenseVectorField" vectorDimension="5" similarityFunction="cosine" knnAlgorithm="hnsw" hnswMaxConnections="8" hnswBeamWidth="46"/>+ <fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswM="10" hnswEfConstruction="40"/>+ <fieldType name="knn_vector2" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswM="6" hnswEfConstruction="60"/>+ <fieldType name="knn_vector3" class="solr.DenseVectorField" vectorDimension="5" similarityFunction="cosine" knnAlgorithm="hnsw" hnswM="8" hnswEfConstruction="46"/><field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>diff --git a/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.javaindex d3790dbbefd..b8e426470f1 100644--- a/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java+++ b/solr/core/src/test/org/apache/solr/schema/DenseVectorFieldTest.java@@ -42,7 +42,7 @@ import org.apache.solr.core.AbstractBadConfigTestBase;import org.apache.solr.handler.loader.JavabinLoader;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.search.neural.KnnQParser;+import org.apache.solr.search.vector.KnnQParser;import org.apache.solr.update.CommitUpdateCommand;import org.apache.solr.update.processor.UpdateRequestProcessor;import org.apache.solr.update.processor.UpdateRequestProcessorChain;@@ -155,8 +155,10 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {assertThat(type1.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));assertThat(type1.getDimension(), is(4));assertThat(type1.getKnnAlgorithm(), is("hnsw"));- assertThat(type1.getHnswMaxConn(), is(10));- assertThat(type1.getHnswBeamWidth(), is(40));+ assertThat(type1.getHnswM(), is(10));+ assertThat(type1.getHnswEfConstruction(), is(40));+ assertThat(type1.getHnswMaxConn(), is(type1.getHnswM()));+ assertThat(type1.getHnswBeamWidth(), is(type1.getHnswEfConstruction()));SchemaField vector2 = schema.getField("vector2");assertNotNull(vector2);@@ -165,8 +167,10 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {assertThat(type2.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));assertThat(type2.getDimension(), is(4));assertThat(type2.getKnnAlgorithm(), is("hnsw"));- assertThat(type2.getHnswMaxConn(), is(6));- assertThat(type2.getHnswBeamWidth(), is(60));+ assertThat(type2.getHnswM(), is(6));+ assertThat(type2.getHnswEfConstruction(), is(60));+ assertThat(type2.getHnswMaxConn(), is(type2.getHnswM()));+ assertThat(type2.getHnswBeamWidth(), is(type2.getHnswEfConstruction()));SchemaField vector3 = schema.getField("vector3");assertNotNull(vector3);@@ -176,8 +180,10 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {assertThat(type3.getDimension(), is(5));assertThat(type3.getKnnAlgorithm(), is("hnsw"));- assertThat(type3.getHnswMaxConn(), is(8));- assertThat(type3.getHnswBeamWidth(), is(46));+ assertThat(type3.getHnswM(), is(8));+ assertThat(type3.getHnswEfConstruction(), is(46));+ assertThat(type3.getHnswMaxConn(), is(type3.getHnswM()));+ assertThat(type3.getHnswBeamWidth(), is(type3.getHnswEfConstruction()));SchemaField vectorDefault = schema.getField("vector_default");assertNotNull(vectorDefault);@@ -186,8 +192,10 @@ public class DenseVectorFieldTest extends AbstractBadConfigTestBase {assertThat(typeDefault.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));assertThat(typeDefault.getKnnAlgorithm(), is("hnsw"));assertThat(typeDefault.getDimension(), is(4));- assertThat(typeDefault.getHnswMaxConn(), is(16));- assertThat(typeDefault.getHnswBeamWidth(), is(100));+ assertThat(typeDefault.getHnswM(), is(16));+ assertThat(typeDefault.getHnswEfConstruction(), is(100));+ assertThat(typeDefault.getHnswMaxConn(), is(typeDefault.getHnswM()));+ assertThat(typeDefault.getHnswBeamWidth(), is(typeDefault.getHnswEfConstruction()));} finally {deleteCore();}diff --git a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java b/solr/core/src/test/org/apache/solr/search/vector/KnnQParserTest.javasimilarity index 99%rename from solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.javarename to solr/core/src/test/org/apache/solr/search/vector/KnnQParserTest.javaindex 32f936dc63a..35920c882c2 100644--- a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java+++ b/solr/core/src/test/org/apache/solr/search/vector/KnnQParserTest.java@@ -14,9 +14,9 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;-import static org.apache.solr.search.neural.KnnQParser.DEFAULT_TOP_K;+import static org.apache.solr.search.vector.KnnQParser.DEFAULT_TOP_K;import java.util.ArrayList;import java.util.Arrays;diff --git a/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java b/solr/core/src/test/org/apache/solr/search/vector/VectorSimilarityQParserTest.javasimilarity index 99%rename from solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.javarename to solr/core/src/test/org/apache/solr/search/vector/VectorSimilarityQParserTest.javaindex c87ed9e8a10..627fb2c176c 100644--- a/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java+++ b/solr/core/src/test/org/apache/solr/search/vector/VectorSimilarityQParserTest.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.search.neural;+package org.apache.solr.search.vector;import java.util.ArrayList;import java.util.Arrays;diff --git a/solr/modules/cuvs/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/modules/cuvs/src/test-files/solr/collection1/conf/solrconfig.xmlindex 38117a9e7a6..30ef530493f 100644--- a/solr/modules/cuvs/src/test-files/solr/collection1/conf/solrconfig.xml+++ b/solr/modules/cuvs/src/test-files/solr/collection1/conf/solrconfig.xml@@ -28,7 +28,14 @@<dataDir>${solr.data.dir:}</dataDir><directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>- <codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory"/>+ <codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory">+ <str name="cuvsWriterThreads">32</str>+ <str name="intGraphDegree">128</str>+ <str name="graphDegree">64</str>+ <str name="hnswLayers">1</str>+ <str name="hnswM">16</str>+ <str name="hnswEfConstruction">100</str>+ </codecFactory><requestHandler name="/select" class="solr.SearchHandler"></requestHandler></config>diff --git a/solr/modules/llm/README.md b/solr/modules/language-models/README.mdsimilarity index 88%rename from solr/modules/llm/README.mdrename to solr/modules/language-models/README.mdindex 2467434a14b..e92a3bd7e41 100644--- a/solr/modules/llm/README.md+++ b/solr/modules/language-models/README.md@@ -15,7 +15,7 @@limitations under the License.-->-The Large Language Model module for Solr provides a set of mechanisms for plugging in third party LLM implementations.+The Language Models module for Solr provides a set of mechanisms for plugging in third party language model implementations.It currently provides text vectorisation through langChain4j.See https://solr.apache.org/guide/solr/latest/query-guide/text-to-vector.html for how to get started.diff --git a/solr/modules/llm/build.gradle b/solr/modules/language-models/build.gradlesimilarity index 100%rename from solr/modules/llm/build.gradlerename to solr/modules/language-models/build.gradlediff --git a/solr/modules/llm/gradle.lockfile b/solr/modules/language-models/gradle.lockfilesimilarity index 100%rename from solr/modules/llm/gradle.lockfilerename to solr/modules/language-models/gradle.lockfilediff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/SolrTextToVectorModel.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/SolrTextToVectorModel.javasimilarity index 96%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/SolrTextToVectorModel.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/SolrTextToVectorModel.javaindex 9a22087b990..21f7f8035be 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/SolrTextToVectorModel.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/SolrTextToVectorModel.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.model;+package org.apache.solr.languagemodels.textvectorisation.model;import dev.langchain4j.data.embedding.Embedding;import dev.langchain4j.model.embedding.EmbeddingModel;@@ -28,8 +28,8 @@ import org.apache.lucene.util.Accountable;import org.apache.lucene.util.RamUsageEstimator;import org.apache.solr.common.SolrException;import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.llm.textvectorisation.store.TextToVectorModelException;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.textvectorisation.store.TextToVectorModelException;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.slf4j.Logger;import org.slf4j.LoggerFactory;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/package-info.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/package-info.javasimilarity index 93%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/package-info.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/package-info.javaindex 3c4fb4d73d6..cadec960174 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/model/package-info.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/model/package-info.java@@ -16,4 +16,4 @@*//** APIs and classes for implementing text to vector logic. */-package org.apache.solr.llm.textvectorisation.model;+package org.apache.solr.languagemodels.textvectorisation.model;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserPlugin.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserPlugin.javasimilarity index 92%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserPlugin.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserPlugin.javaindex 3780da9972c..3c213928cb3 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserPlugin.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserPlugin.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.search;+package org.apache.solr.languagemodels.textvectorisation.search;import java.io.IOException;import java.util.Arrays;@@ -26,18 +26,18 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.rest.ManagedResource;import org.apache.solr.rest.ManagedResourceObserver;import org.apache.solr.search.QParser;import org.apache.solr.search.QParserPlugin;import org.apache.solr.search.SyntaxError;-import org.apache.solr.search.neural.KnnQParser;+import org.apache.solr.search.vector.KnnQParser;/**- * A neural query parser that encode the query to a vector and then run K-nearest neighbors search+ * A vector query parser that encode the query to a vector and then run K-nearest neighbors search* on Dense Vector fields. See Wiki page* https://solr.apache.org/guide/solr/latest/query-guide/dense-vector-search.html*/diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/package-info.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/package-info.javasimilarity index 93%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/package-info.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/package-info.javaindex f188323c2c3..04aebea7789 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/search/package-info.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/search/package-info.java@@ -16,4 +16,4 @@*//** APIs and classes for implementing text to vector QueryParsers. */-package org.apache.solr.llm.textvectorisation.search;+package org.apache.solr.languagemodels.textvectorisation.search;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelException.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelException.javasimilarity index 94%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelException.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelException.javaindex d64b124ec5c..8709ebf6929 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelException.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelException.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.store;+package org.apache.solr.languagemodels.textvectorisation.store;public class TextToVectorModelException extends RuntimeException {diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelStore.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelStore.javasimilarity index 93%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelStore.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelStore.javaindex e971e050957..7d24d25f57e 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/TextToVectorModelStore.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/TextToVectorModelStore.java@@ -14,14 +14,14 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.store;+package org.apache.solr.languagemodels.textvectorisation.store;import java.util.ArrayList;import java.util.Collections;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;/** Simple store to manage CRUD operations on the {@link SolrTextToVectorModel} */public class TextToVectorModelStore {diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/package-info.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/package-info.javasimilarity index 92%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/package-info.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/package-info.javaindex 36303a4c076..5e79341f992 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/package-info.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/package-info.java@@ -16,4 +16,4 @@*//** Contains model store related classes. */-package org.apache.solr.llm.textvectorisation.store;+package org.apache.solr.languagemodels.textvectorisation.store;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/ManagedTextToVectorModelStore.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/ManagedTextToVectorModelStore.javasimilarity index 95%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/ManagedTextToVectorModelStore.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/ManagedTextToVectorModelStore.javaindex 2ae76c45017..70c03ffc47e 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/ManagedTextToVectorModelStore.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/ManagedTextToVectorModelStore.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.store.rest;+package org.apache.solr.languagemodels.textvectorisation.store.rest;import java.lang.invoke.MethodHandles;import java.util.LinkedHashMap;@@ -26,9 +26,9 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;-import org.apache.solr.llm.textvectorisation.store.TextToVectorModelException;-import org.apache.solr.llm.textvectorisation.store.TextToVectorModelStore;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.store.TextToVectorModelException;+import org.apache.solr.languagemodels.textvectorisation.store.TextToVectorModelStore;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.rest.BaseSolrResource;import org.apache.solr.rest.ManagedResource;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/package-info.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/package-info.javasimilarity index 92%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/package-info.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/package-info.javaindex 118bc245e30..ca70f6d9611 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/store/rest/package-info.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/store/rest/package-info.java@@ -16,4 +16,4 @@*//** Contains the {@link org.apache.solr.rest.ManagedResource} that encapsulate the model stores. */-package org.apache.solr.llm.textvectorisation.store.rest;+package org.apache.solr.languagemodels.textvectorisation.store.rest;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessor.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessor.javasimilarity index 95%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessor.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessor.javaindex cac6d4283aa..f20d91e9685 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessor.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessor.java@@ -15,7 +15,7 @@* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.update.processor;+package org.apache.solr.languagemodels.textvectorisation.update.processor;import java.io.IOException;import java.lang.invoke.MethodHandles;@@ -23,7 +23,7 @@ import java.util.ArrayList;import java.util.List;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.SolrInputField;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.SchemaField;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.javasimilarity index 94%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.javaindex c4f7ea2924f..e524b689d56 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.java@@ -15,14 +15,14 @@* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.update.processor;+package org.apache.solr.languagemodels.textvectorisation.update.processor;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.RequiredSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.schema.DenseVectorField;diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/package-info.java b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/package-info.javasimilarity index 91%rename from solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/package-info.javarename to solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/package-info.javaindex 2c5f2ef0072..877a4fa081b 100644--- a/solr/modules/llm/src/java/org/apache/solr/llm/textvectorisation/update/processor/package-info.java+++ b/solr/modules/language-models/src/java/org/apache/solr/languagemodels/textvectorisation/update/processor/package-info.java@@ -16,4 +16,4 @@*//** Contains update request processor related classes. */-package org.apache.solr.llm.textvectorisation.update.processor;+package org.apache.solr.languagemodels.textvectorisation.update.processor;diff --git a/solr/modules/llm/src/java/overview.html b/solr/modules/language-models/src/java/overview.htmlsimilarity index 100%rename from solr/modules/llm/src/java/overview.htmlrename to solr/modules/language-models/src/java/overview.htmldiff --git a/solr/modules/llm/src/test-files/log4j2.xml b/solr/modules/language-models/src/test-files/log4j2.xmlsimilarity index 100%rename from solr/modules/llm/src/test-files/log4j2.xmlrename to solr/modules/language-models/src/test-files/log4j2.xmldiff --git a/solr/modules/llm/src/test-files/modelExamples/cohere-model.json b/solr/modules/language-models/src/test-files/modelExamples/cohere-model.jsonsimilarity index 100%rename from solr/modules/llm/src/test-files/modelExamples/cohere-model.jsonrename to solr/modules/language-models/src/test-files/modelExamples/cohere-model.jsondiff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json b/solr/modules/language-models/src/test-files/modelExamples/dummy-model-ambiguous.jsonsimilarity index 53%rename from solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.jsonrename to solr/modules/language-models/src/test-files/modelExamples/dummy-model-ambiguous.jsonindex 417ce5b11a6..c9fb4dc48db 100644--- a/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json+++ b/solr/modules/language-models/src/test-files/modelExamples/dummy-model-ambiguous.json@@ -1,5 +1,5 @@{- "class": "org.apache.solr.llm.textvectorisation.model.DummyEmbeddingModel",+ "class": "org.apache.solr.languagemodels.textvectorisation.model.DummyEmbeddingModel","name": "dummy-1","params": {"embedding": [1.0, 2.0, 3.0, 4.0],diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json b/solr/modules/language-models/src/test-files/modelExamples/dummy-model-unsupported.jsonsimilarity index 53%rename from solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.jsonrename to solr/modules/language-models/src/test-files/modelExamples/dummy-model-unsupported.jsonindex 7316cd7dd03..cddbacfdac3 100644--- a/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json+++ b/solr/modules/language-models/src/test-files/modelExamples/dummy-model-unsupported.json@@ -1,5 +1,5 @@{- "class": "org.apache.solr.llm.textvectorisation.model.DummyEmbeddingModel",+ "class": "org.apache.solr.languagemodels.textvectorisation.model.DummyEmbeddingModel","name": "dummy-1","params": {"embedding": [1.0, 2.0, 3.0, 4.0],diff --git a/solr/modules/language-models/src/test-files/modelExamples/dummy-model.json b/solr/modules/language-models/src/test-files/modelExamples/dummy-model.jsonnew file mode 100644index 00000000000..115766e8612--- /dev/null+++ b/solr/modules/language-models/src/test-files/modelExamples/dummy-model.json@@ -0,0 +1,7 @@+{+ "class": "org.apache.solr.languagemodels.textvectorisation.model.DummyEmbeddingModel",+ "name": "dummy-1",+ "params": {+ "embedding": [1.0, 2.0, 3.0, 4.0]+ }+}diff --git a/solr/modules/language-models/src/test-files/modelExamples/exception-throwing-model.json b/solr/modules/language-models/src/test-files/modelExamples/exception-throwing-model.jsonnew file mode 100644index 00000000000..76e704f3ac4--- /dev/null+++ b/solr/modules/language-models/src/test-files/modelExamples/exception-throwing-model.json@@ -0,0 +1,6 @@+{+ "class": "org.apache.solr.languagemodels.textvectorisation.model.ExceptionThrowingEmbeddingModel",+ "name": "exception-throwing-model",+ "params": {+ }+}diff --git a/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json b/solr/modules/language-models/src/test-files/modelExamples/huggingface-model.jsonsimilarity index 100%rename from solr/modules/llm/src/test-files/modelExamples/huggingface-model.jsonrename to solr/modules/language-models/src/test-files/modelExamples/huggingface-model.jsondiff --git a/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json b/solr/modules/language-models/src/test-files/modelExamples/mistralai-model.jsonsimilarity index 100%rename from solr/modules/llm/src/test-files/modelExamples/mistralai-model.jsonrename to solr/modules/language-models/src/test-files/modelExamples/mistralai-model.jsondiff --git a/solr/modules/llm/src/test-files/modelExamples/openai-model.json b/solr/modules/language-models/src/test-files/modelExamples/openai-model.jsonsimilarity index 100%rename from solr/modules/llm/src/test-files/modelExamples/openai-model.jsonrename to solr/modules/language-models/src/test-files/modelExamples/openai-model.jsondiff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml b/solr/modules/language-models/src/test-files/solr/collection1/conf/schema.xmlsimilarity index 100%rename from solr/modules/llm/src/test-files/solr/collection1/conf/schema.xmlrename to solr/modules/language-models/src/test-files/solr/collection1/conf/schema.xmldiff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml b/solr/modules/language-models/src/test-files/solr/collection1/conf/solrconfig-language-models.xmlsimilarity index 87%rename from solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xmlrename to solr/modules/language-models/src/test-files/solr/collection1/conf/solrconfig-language-models.xmlindex 005098bda8b..f9c58a3a849 100644--- a/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml+++ b/solr/modules/language-models/src/test-files/solr/collection1/conf/solrconfig-language-models.xml@@ -21,9 +21,9 @@<requestParsers /></requestDispatcher>- <!-- Query parser used to run neural queries-->+ <!-- Query parser used to run vector search queries--><queryParser name="knn_text_to_vector"- class="org.apache.solr.llm.textvectorisation.search.TextToVectorQParserPlugin" />+ class="org.apache.solr.languagemodels.textvectorisation.search.TextToVectorQParserPlugin" /><query><filterCache class="solr.CaffeineCache" size="4096"@@ -43,7 +43,7 @@<str name="dir">${solr.data.dir:}</str></updateLog></updateHandler>-+<!-- Query request handler managing models and features --><requestHandler name="/query" class="solr.SearchHandler"><lst name="defaults">@@ -55,7 +55,7 @@</requestHandler><updateRequestProcessorChain name="textToVector">- <processor class="solr.llm.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory">+ <processor class="solr.languagemodels.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory"><str name="inputField">_text_</str><str name="outputField">vector</str><str name="model">dummy-1</str>@@ -64,14 +64,14 @@</updateRequestProcessorChain><updateRequestProcessorChain name="failingTextToVector">- <processor class="solr.llm.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory">+ <processor class="solr.languagemodels.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory"><str name="inputField">_text_</str><str name="outputField">vector</str><str name="model">exception-throwing-model</str></processor><processor class="solr.RunUpdateProcessorFactory"/></updateRequestProcessorChain>--++</config>diff --git a/solr/modules/llm/src/test-files/solr/solr.xml b/solr/modules/language-models/src/test-files/solr/solr.xmlsimilarity index 100%rename from solr/modules/llm/src/test-files/solr/solr.xmlrename to solr/modules/language-models/src/test-files/solr/solr.xmldiff --git a/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/TestLanguageModelBase.javasimilarity index 94%rename from solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/TestLanguageModelBase.javaindex bd79b2dcbc3..5c8d10ac926 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/TestLanguageModelBase.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm;+package org.apache.solr.languagemodels;import java.lang.invoke.MethodHandles;import java.net.URL;@@ -26,12 +26,12 @@ import java.util.Arrays;import java.util.List;import org.apache.commons.io.file.PathUtils;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.apache.solr.util.RestTestBase;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-public class TestLlmBase extends RestTestBase {+public class TestLanguageModelBase extends RestTestBase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -91,7 +91,7 @@ public class TestLlmBase extends RestTestBase {}public static void loadModel(String fileName, String status) throws Exception {- final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName);+ final URL url = TestLanguageModelBase.class.getResource("/modelExamples/" + fileName);final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8);assertJPut(@@ -101,7 +101,7 @@ public class TestLlmBase extends RestTestBase {}public static void loadModel(String fileName) throws Exception {- final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName);+ final URL url = TestLanguageModelBase.class.getResource("/modelExamples/" + fileName);final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8);assertJPut(diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModel.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModel.javasimilarity index 97%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModel.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModel.javaindex 049d1cded7e..cbb966348f0 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModel.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModel.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.model;+package org.apache.solr.languagemodels.textvectorisation.model;import dev.langchain4j.data.embedding.Embedding;import dev.langchain4j.data.segment.TextSegment;diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModelTest.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModelTest.javasimilarity index 96%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModelTest.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModelTest.javaindex 10f5aae04a3..0948af705eb 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/DummyEmbeddingModelTest.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/DummyEmbeddingModelTest.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.model;+package org.apache.solr.languagemodels.textvectorisation.model;import org.apache.solr.SolrTestCase;import org.junit.Test;diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/ExceptionThrowingEmbeddingModel.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/ExceptionThrowingEmbeddingModel.javasimilarity index 96%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/ExceptionThrowingEmbeddingModel.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/ExceptionThrowingEmbeddingModel.javaindex 7c69ed8352d..c0c3b52bf39 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/model/ExceptionThrowingEmbeddingModel.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/model/ExceptionThrowingEmbeddingModel.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.model;+package org.apache.solr.languagemodels.textvectorisation.model;import dev.langchain4j.data.embedding.Embedding;import dev.langchain4j.data.segment.TextSegment;diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserTest.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserTest.javasimilarity index 98%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserTest.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserTest.javaindex 4789093d601..bfe42e30842 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/search/TextToVectorQParserTest.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/search/TextToVectorQParserTest.java@@ -14,20 +14,20 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.search;+package org.apache.solr.languagemodels.textvectorisation.search;import java.util.Arrays;import java.util.Locale;import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.llm.TestLlmBase;+import org.apache.solr.languagemodels.TestLanguageModelBase;import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;-public class TextToVectorQParserTest extends TestLlmBase {+public class TextToVectorQParserTest extends TestLanguageModelBase {@BeforeClasspublic static void init() throws Exception {- setupTest("solrconfig-llm.xml", "schema.xml", true, false);+ setupTest("solrconfig-language-models.xml", "schema.xml", true, false);loadModel("dummy-model.json");}diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManager.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManager.javasimilarity index 96%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManager.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManager.javaindex 05e7f3bb0e9..da70dbb696f 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManager.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManager.java@@ -14,12 +14,12 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.store.rest;+package org.apache.solr.languagemodels.textvectorisation.store.rest;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrResourceLoader;-import org.apache.solr.llm.TestLlmBase;-import org.apache.solr.llm.textvectorisation.search.TextToVectorQParserPlugin;+import org.apache.solr.languagemodels.TestLanguageModelBase;+import org.apache.solr.languagemodels.textvectorisation.search.TextToVectorQParserPlugin;import org.apache.solr.rest.ManagedResource;import org.apache.solr.rest.ManagedResourceStorage;import org.apache.solr.rest.RestManager;@@ -27,11 +27,11 @@ import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;-public class TestModelManager extends TestLlmBase {+public class TestModelManager extends TestLanguageModelBase {@BeforeClasspublic static void init() throws Exception {- setupTest("solrconfig-llm.xml", "schema.xml", false, false);+ setupTest("solrconfig-language-models.xml", "schema.xml", false, false);}@AfterClassdiff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManagerPersistence.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManagerPersistence.javasimilarity index 94%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManagerPersistence.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManagerPersistence.javaindex a7ed7923eeb..efa90848a9c 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/store/rest/TestModelManagerPersistence.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/store/rest/TestModelManagerPersistence.java@@ -14,23 +14,23 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.store.rest;+package org.apache.solr.languagemodels.textvectorisation.store.rest;import static java.nio.charset.StandardCharsets.UTF_8;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import org.apache.solr.common.util.Utils;-import org.apache.solr.llm.TestLlmBase;+import org.apache.solr.languagemodels.TestLanguageModelBase;import org.junit.After;import org.junit.Before;import org.junit.Test;-public class TestModelManagerPersistence extends TestLlmBase {+public class TestModelManagerPersistence extends TestLanguageModelBase {@Beforepublic void init() throws Exception {- setupTest("solrconfig-llm.xml", "schema.xml", false, true);+ setupTest("solrconfig-language-models.xml", "schema.xml", false, true);}@Afterdiff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.javasimilarity index 94%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.javaindex c5b7991b3c5..4185d9913b2 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactoryTest.java@@ -14,15 +14,15 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.update.processor;+package org.apache.solr.languagemodels.textvectorisation.update.processor;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrCore;-import org.apache.solr.llm.TestLlmBase;-import org.apache.solr.llm.textvectorisation.model.SolrTextToVectorModel;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.TestLanguageModelBase;+import org.apache.solr.languagemodels.textvectorisation.model.SolrTextToVectorModel;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.apache.solr.request.SolrQueryRequestBase;import org.apache.solr.update.processor.UpdateRequestProcessor;import org.junit.After;@@ -31,11 +31,11 @@ import org.junit.Before;import org.junit.BeforeClass;import org.junit.Test;-public class TextToVectorUpdateProcessorFactoryTest extends TestLlmBase {+public class TextToVectorUpdateProcessorFactoryTest extends TestLanguageModelBase {@BeforeClasspublic static void init() throws Exception {- setupTest("solrconfig-llm.xml", "schema.xml", false, false);+ setupTest("solrconfig-language-models.xml", "schema.xml", false, false);}@AfterClassdiff --git a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.java b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.javasimilarity index 94%rename from solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.javarename to solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.javaindex c82d2312977..bc7de75748b 100644--- a/solr/modules/llm/src/test/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.java+++ b/solr/modules/language-models/src/test/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorTest.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.solr.llm.textvectorisation.update.processor;+package org.apache.solr.languagemodels.textvectorisation.update.processor;import java.io.IOException;import org.apache.solr.client.solrj.RemoteSolrException;@@ -22,18 +22,18 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.SolrQuery;import org.apache.solr.client.solrj.request.UpdateRequest;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.llm.TestLlmBase;-import org.apache.solr.llm.textvectorisation.store.rest.ManagedTextToVectorModelStore;+import org.apache.solr.languagemodels.TestLanguageModelBase;+import org.apache.solr.languagemodels.textvectorisation.store.rest.ManagedTextToVectorModelStore;import org.junit.After;import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;-public class TextToVectorUpdateProcessorTest extends TestLlmBase {+public class TextToVectorUpdateProcessorTest extends TestLanguageModelBase {@BeforeClasspublic static void init() throws Exception {- setupTest("solrconfig-llm.xml", "schema.xml", false, false);+ setupTest("solrconfig-language-models.xml", "schema.xml", false, false);}@AfterClassdiff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model.jsondeleted file mode 100644index 750344f37c5..00000000000--- a/solr/modules/llm/src/test-files/modelExamples/dummy-model.json+++ /dev/null@@ -1,7 +0,0 @@-{- "class": "org.apache.solr.llm.textvectorisation.model.DummyEmbeddingModel",- "name": "dummy-1",- "params": {- "embedding": [1.0, 2.0, 3.0, 4.0]- }-}diff --git a/solr/modules/llm/src/test-files/modelExamples/exception-throwing-model.json b/solr/modules/llm/src/test-files/modelExamples/exception-throwing-model.jsondeleted file mode 100644index c058da250d7..00000000000--- a/solr/modules/llm/src/test-files/modelExamples/exception-throwing-model.json+++ /dev/null@@ -1,6 +0,0 @@-{- "class": "org.apache.solr.llm.textvectorisation.model.ExceptionThrowingEmbeddingModel",- "name": "exception-throwing-model",- "params": {- }-}diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adocindex ae8ea11f604..c2377045ce1 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc@@ -415,10 +415,10 @@ When using any of these factories, please consult the {solr-javadocs}/core/org/aThese processors are included in Solr releases as "module", and require additional jars loaded at runtime.See the README files associated with each module for details:-The {solr-javadocs}/modules/llm/index.html[`llm`] module provides::+The {solr-javadocs}/modules/language-models/index.html[`language-models`] module provides::-{solr-javadocs}/modules/llm/org/apache/solr/llm/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.html[TextToVectorUpdateProcessorFactory]:: Update processor which vectorises a textual field in input and adds the resulting vector as the value of a new field.-It uses external text to vectors LLM to perform the vectorisation for each processed document.+{solr-javadocs}/modules/language-models/org/apache/solr/languagemodels/textvectorisation/update/processor/TextToVectorUpdateProcessorFactory.html[TextToVectorUpdateProcessorFactory]:: Update processor which vectorises a textual field in input and adds the resulting vector as the value of a new field.+It uses external text to vectors language models to perform the vectorisation for each processed document.For more information: xref:query-guide:text-to-vector.adoc[Update Request Processor]The {solr-javadocs}/modules/langid/index.html[`langid`] module provides::diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adocindex 26bbcff1670..87eb6941ba9 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc@@ -103,13 +103,16 @@ the cosine similarity scores returned by Solr are normalized like this : `(1 + c[NOTE]the preferred way to perform cosine similarity is to normalize all vectors to unit length, and instead use DOT_PRODUCT. You should only use this function if you need to preserve the original vectors and cannot normalize them in advance.+[NOTE]+The HNSW parameters `hnswM` and `hnswEfConstruction`, previously known as `hnswMaxConnections` and `hnswBeamWidth` respectively.+To use the following advanced parameters that customise the codec formatand the hyperparameter of the HNSW algorithm, make sure the xref:configuration-guide:codec-factory.adoc[Schema Codec Factory], is in use.Here's how `DenseVectorField` can be configured with the advanced hyperparameters:[source,xml]-<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswMaxConnections="10" hnswBeamWidth="40"/>+<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="hnsw" hnswM="10" hnswEfConstruction="40"/><field name="vector" type="knn_vector" indexed="true" stored="true"/>`knnAlgorithm`::@@ -139,7 +142,7 @@ Please note that the `knnAlgorithm` accepted values may change in future releaseAccepted values: `FLOAT32`, `BYTE`.-`hnswMaxConnections`::+`hnswM`::+[%autowidth,frame=none]|===@@ -150,12 +153,12 @@ Accepted values: `FLOAT32`, `BYTE`.+Controls how many of the nearest neighbor candidates are connected to the new node.+-It has the same meaning as `M` from the 2018 paper.+For more details, refer to the official https://arxiv.org/pdf/1603.09320[2018 paper], where this parameter `M` is defined.+Accepted values:Any integer.-`hnswBeamWidth`::+`hnswEfConstruction`::+[%autowidth,frame=none]|===@@ -166,7 +169,7 @@ Any integer.+It is the number of nearest neighbor candidates to track while searching the graph for each newly inserted node.+-It has the same meaning as `efConstruction` from the 2018 paper.+For more details, refer to the official https://arxiv.org/pdf/1603.09320[2018 paper], where this parameter `efConstruction` is defined.+Accepted values:Any integer.@@ -718,7 +721,21 @@ Define the `fieldType` in the schema, with knnAlgorithm set to `cagra_hnsw`:[source,xml]-----<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="8" knnAlgorithm="cagra_hnsw" similarityFunction="cosine" cuvsWriterThreads="32" cuvsIntGraphDegree="128" cuvsGraphDegree="64" cuvsHnswLayers="1" cuvsHnswM="16" cuvsHNSWEfConstruction="100"/>+<fieldType name="knn_vector" class="solr.DenseVectorField" vectorDimension="8" knnAlgorithm="cagra_hnsw" similarityFunction="cosine" />+----++Define the xref:configuration-guide:codec-factory.adoc[codecFactory] in xref:configuration-guide:configuring-solrconfig-xml.adoc[solrconfig.xml]++[source,xml]+----+<codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory">+ <str name="cuvsWriterThreads">8</str>+ <str name="intGraphDegree">128</str>+ <str name="graphDegree">64</str>+ <str name="hnswLayers">1</str>+ <str name="hnswM">16</str>+ <str name="hnswEfConstruction">100</str>+</codecFactory>----Where:@@ -731,16 +748,9 @@ Where:* `cuvsHnswLayers` - Number of HNSW graph layers to construct while building the HNSW index-* `cuvsHnswM` - Max connections parameter passed to the fallback Lucene99HnswVectorsWriter--* `cuvsHNSWEfConstruction` - Beam width parameter passed to the fallback Lucene99HnswVectorsWriter--Define the xref:configuration-guide:codec-factory.adoc[codecFactory] in xref:configuration-guide:configuring-solrconfig-xml.adoc[solrconfig.xml]+* `hnswM` - hnswM parameter passed to the fallback Lucene99HnswVectorsWriter-[source,xml]------<codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory"/>-----+* `hnswEfConstruction` - hnswEfConstruction parameter passed to the fallback Lucene99HnswVectorsWriter=== Example@@ -873,7 +883,14 @@ cat > cuvs_configset/conf/solrconfig.xml << 'EOF'</autoSoftCommit></updateHandler>- <codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory"/>+ <codecFactory name="CuVSCodecFactory" class="org.apache.solr.cuvs.CuVSCodecFactory">+ <str name="cuvsWriterThreads">32</str>+ <str name="intGraphDegree">128</str>+ <str name="graphDegree">64</str>+ <str name="hnswLayers">1</str>+ <str name="hnswM">16</str>+ <str name="hnswEfConstruction">100</str>+ </codecFactory><requestHandler name="/select" class="solr.SearchHandler"><lst name="defaults">diff --git a/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adocindex 16c3c20e6a9..59f55acfedf 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/other-parsers.adoc@@ -1004,8 +1004,8 @@ For more information about the possibilities of nested queries, see Yonik Seeley== Vector Query Parsers-There are currently three Query Parsers in Solr to provide Vector Search:+There are currently three Query Parsers in Solr to provide Vector Search:- `knn`: matches k-nearest neighbours (knn) documents to the target vector.- `knn_text_to_vector`: encode a textual query to a vector using a dedicated Language Model and matches knn documents to such query vector.- `vectorSimilarity`: matches documents whose similarity with the target vector is a above a minimum threshold.diff --git a/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc b/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adocindex cf0545ed030..13d845459f7 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/text-to-vector.adoc@@ -16,11 +16,11 @@// specific language governing permissions and limitations// under the License.-This module brings the power of *Large Language Models* (*LLM*s) to Solr.+This module brings the power of *Language Models* to Solr.More specifically, it provides a text-to-vector capability, used on documents or queries, via integrating with popular external services that do this.-The state-of-the-art of such services use an LLM, hence the name of this module.+The state-of-the-art of such services use a Language Model, hence the name of this module._Without_ this module, vectors must be supplied _to_ Solr for indexing & searching, possibly coordinating with such services.@@ -35,9 +35,9 @@ Often this process is called 'text embedding' as it projects a piece of text intVector distance metrics (algorithms) can then be used to compute a pairwise similarity, producing a score.-=== Large Language Models+=== Language Models-Specific Large Language Models are able to encode text to a numerical vector.+Specific Language Models are able to encode text to a numerical vector.These models are often called Embedding Models as they encode text to vector embeddings.@@ -53,7 +53,7 @@ Apache Solr uses https://github.com/langchain4j/langchain4j[LangChain4j] to conn[IMPORTANT]====-This module sends your documents and queries off to some hosted service on the internet.+This module sends your documents and queries off to some hosted service on the internet.There are cost, privacy, performance, and service availability implications on such a strong dependency that should be diligently examined before employing this module in a serious way.====@@ -68,17 +68,17 @@ If you want to add support for additional services or improve the support for th== Module-This is provided via the `llm` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use.+This is provided via the `language-models` xref:configuration-guide:solr-modules.adoc[Solr Module] that needs to be enabled before use.-== LLM Configuration+== Language Model Configuration-You need to register / configure the plugins provided by the LLM module that you want to use. This is done in `solrconfig.xml`.+You need to register / configure the plugins provided by the Language Models module that you want to use. This is done in `solrconfig.xml`.* Declaration of the `knn_text_to_vector` query parser.+[source,xml]-----<queryParser name="knn_text_to_vector" class="org.apache.solr.llm.textvectorisation.search.TextToVectorQParserPlugin"/>+<queryParser name="knn_text_to_vector" class="org.apache.solr.languagemodels.textvectorisation.search.TextToVectorQParserPlugin"/>----@@ -107,7 +107,7 @@ s|Required |Default: none|===+The model implementation.-Accepted values:+Accepted values:* `dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel`.* `dev.langchain4j.model.mistralai.MistralAiEmbeddingModel`.@@ -278,7 +278,7 @@ To vectorise textual fields of your documents at indexing time you need to confi[source,xml]----<updateRequestProcessorChain name="textToVector">- <processor class="solr.llm.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory">+ <processor class="solr.languagemodels.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory"><str name="inputField">_text_</str><str name="outputField">vector</str><str name="model">dummy-1</str>@@ -332,7 +332,7 @@ This can be done in Solr defining two update request processors chains: one that<processor class="solr.processorN">...</processor>-<processor class="solr.llm.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory">+<processor class="solr.languagemodels.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory"><str name="inputField">_text_</str><str name="outputField">vector</str><str name="model">dummy-1</str>@@ -370,7 +370,7 @@ You still define two chains, but this time the 'vectorisation' one only includes[source,xml]----<updateRequestProcessorChain name="vectorisation">-<processor class="solr.llm.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory">+<processor class="solr.languagemodels.textvectorisation.update.processor.TextToVectorUpdateProcessorFactory"><str name="inputField">_text_</str><str name="outputField">vector</str><str name="model">dummy-1</str>@@ -411,4 +411,4 @@ To run a query that vectorises your query text, using a model you previously uplThe search results retrieved are the k=10 nearest documents to the vector encoded from the query `hello world query`, using the model `a-model`.-For more details on how to work with vector search query parsers in Apache Solr, please refer to the dedicated page: xref:dense-vector-search.adoc[Dense Vector Search]\ No newline at end of file+For more details on how to work with vector search query parsers in Apache Solr, please refer to the dedicated page: xref:dense-vector-search.adoc[Dense Vector Search]diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adocindex f3f7b2dc379..c6e61e70969 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc@@ -271,3 +271,9 @@ Due to Lucene 10 changes (https://github.com/apache/lucene/pull/12875), `PathHie</analyzer></fieldType>----++=== Renaming in Vector Search+Attention:++* The `llm` module has been renamed to `language-models`.+* The HNSW parameters `hnswMaxConnections` and `hnswBeamWidth` have been renamed to `hnswM` and `hnswEfConstruction`, respectively, so they must be updated accordingly in the schema.xml file.