Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17023: Use Modern NLP Models via ONNX and Apache OpenNLP with Solr (#1999)

Eric Pugh08c07309d878
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 6c2d83df338..e9760f446cc 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -38,7 +38,7 @@ apache-httpcomponents-httpmime = "4.5.14"
apache-kafka = "3.9.0"
apache-log4j = "2.21.0"
apache-lucene = "10.2.1"
-apache-opennlp = "2.5.3"
+apache-opennlp = "2.5.4"
apache-poi = "5.2.2"
apache-rat = "0.15"
apache-tika = "1.28.5"
@@ -171,6 +171,7 @@ nlopez-compose = "0.4.27"
nodegradle-node = "7.1.0"
# @keep Node JS version used in node.gradle (LTS)
nodejs = "22.18.0"
+onnx = "1.20.0"
openapi = "7.14.0"
openjdk-jmh = "1.37"
opentelemetry = "1.53.0"
@@ -285,6 +286,7 @@ apache-lucene-sandbox = { module = "org.apache.lucene:lucene-sandbox", version.r
apache-lucene-spatialextras = { module = "org.apache.lucene:lucene-spatial-extras", version.ref = "apache-lucene" }
apache-lucene-suggest = { module = "org.apache.lucene:lucene-suggest", version.ref = "apache-lucene" }
apache-lucene-testframework = { module = "org.apache.lucene:lucene-test-framework", version.ref = "apache-lucene" }
+apache-opennlp-dl = { module = "org.apache.opennlp:opennlp-dl", version.ref = "apache-opennlp" }
apache-opennlp-tools = { module = "org.apache.opennlp:opennlp-tools", version.ref = "apache-opennlp" }
apache-poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "apache-poi" }
apache-poi-poi = { module = "org.apache.poi:poi", version.ref = "apache-poi" }
@@ -473,6 +475,7 @@ netty-transport-classes-epoll = { module = "io.netty:netty-transport-classes-epo
netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" }
nimbusds-josejwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbusds-josejwt" }
nlopez-compose-ktlintrules = { module = "io.nlopez.compose.rules:ktlint", version.ref = "nlopez-compose" }
+onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnx" }
openjdk-jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "openjdk-jmh" }
openjdk-jmh-generatorannprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "openjdk-jmh" }
opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" }
diff --git a/solr/core/gradle.lockfile b/solr/core/gradle.lockfile
index 0079eb41880..6cd8dff00f6 100644
--- a/solr/core/gradle.lockfile
+++ b/solr/core/gradle.lockfile
@@ -33,6 +33,7 @@ com.google.protobuf:protobuf-java:3.25.8=annotationProcessor,errorprone,testAnno
com.ibm.icu:icu4j:77.1=jarValidation,testRuntimeClasspath
com.j256.simplemagic:simplemagic:1.17=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
com.jayway.jsonpath:json-path:2.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+com.microsoft.onnxruntime:onnxruntime:1.20.0=jarValidation,testRuntimeClasspath
com.tdunning:t-digest:3.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
commons-cli:commons-cli:1.10.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
commons-codec:commons-codec:1.19.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
@@ -114,7 +115,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=compileClasspath,jarValidation,ru
org.apache.lucene:lucene-spatial3d:10.2.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.lucene:lucene-suggest:10.2.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.opennlp:opennlp-tools:2.5.3=jarValidation,testRuntimeClasspath
+org.apache.opennlp:opennlp-dl:2.5.4=jarValidation,testRuntimeClasspath
+org.apache.opennlp:opennlp-tools:2.5.4=jarValidation,testRuntimeClasspath
org.apache.zookeeper:zookeeper-jute:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apiguardian:apiguardian-api:1.1.2=jarValidation,testCompileClasspath,testRuntimeClasspath
diff --git a/solr/licenses/onnxruntime-1.20.0.jar.sha1 b/solr/licenses/onnxruntime-1.20.0.jar.sha1
new file mode 100644
index 00000000000..a1973ae8b1c
--- /dev/null
+++ b/solr/licenses/onnxruntime-1.20.0.jar.sha1
@@ -0,0 +1 @@
+717cd187f7d5080a8cc5c8aa761cafe2db028de1
diff --git a/solr/licenses/onnxruntime-LICENSE-MIT.txt b/solr/licenses/onnxruntime-LICENSE-MIT.txt
new file mode 100644
index 00000000000..48bc6bb4996
--- /dev/null
+++ b/solr/licenses/onnxruntime-LICENSE-MIT.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/solr/licenses/opennlp-LICENSE-ASL.txt b/solr/licenses/opennlp-LICENSE-ASL.txt
new file mode 100644
index 00000000000..d1f4d5cc086
--- /dev/null
+++ b/solr/licenses/opennlp-LICENSE-ASL.txt
@@ -0,0 +1,314 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+EXTERNAL COMPONENTS
+
+Apache PDFBox includes a number of components with separate copyright notices
+and license terms. Your use of these components is subject to the terms and
+conditions of the following licenses.
+
+Contributions made to the original PDFBox project:
+
+ Copyright (c) 2002-2007, www.pdfbox.org
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of pdfbox; nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+
+Adobe Font Metrics (AFM) for PDF Core 14 Fonts
+
+ This file and the 14 PostScript(R) AFM files it accompanies may be used,
+ copied, and distributed for any purpose and without charge, with or without
+ modification, provided that all copyright notices are retained; that the
+ AFM files are not distributed without this file; that all modifications
+ to this file or any of the AFM files are prominently noted in the modified
+ file(s); and that this paragraph is not modified. Adobe Systems has no
+ responsibility or obligation to support the use of the AFM files.
+
+CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads)
+
+ Copyright 1990-2009 Adobe Systems Incorporated.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ Neither the name of Adobe Systems Incorporated nor the names of its
+ contributors may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ THE POSSIBILITY OF SUCH DAMAGE.
+
+Glyphlist (http://www.adobe.com/devnet/opentype/archives/glyph.html)
+
+ Copyright (c) 1997,1998,2002,2007 Adobe Systems Incorporated
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this documentation file to use, copy, publish, distribute,
+ sublicense, and/or sell copies of the documentation, and to permit
+ others to do the same, provided that:
+ - No modification, editing or other alteration of this document is
+ allowed; and
+ - The above copyright notice and this permission notice shall be
+ included in all copies of the documentation.
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this documentation file, to create their own derivative works
+ from the content of this document to use, copy, publish, distribute,
+ sublicense, and/or sell the derivative works, and to permit others to do
+ the same, provided that the derived work is not represented as being a
+ copy or version of this document.
+
+ Adobe shall not be liable to any party for any loss of revenue or profit
+ or for indirect, incidental, special, consequential, or other similar
+ damages, whether based on tort (including without limitation negligence
+ or strict liability), contract or other legal or equitable grounds even
+ if Adobe has been advised or had reason to know of the possibility of
+ such damages. The Adobe materials are provided on an "AS IS" basis.
+ Adobe specifically disclaims all express, statutory, or implied
+ warranties relating to the Adobe materials, including but not limited to
+ those concerning merchantability or fitness for a particular purpose or
+ non-infringement of any third party rights regarding the Adobe
+ materials.
+
diff --git a/solr/licenses/opennlp-NOTICE.txt b/solr/licenses/opennlp-NOTICE.txt
new file mode 100644
index 00000000000..617c7d9b106
--- /dev/null
+++ b/solr/licenses/opennlp-NOTICE.txt
@@ -0,0 +1,101 @@
+Apache OpenNLP
+Copyright 2021-2025 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+============================================================================
+
+The snowball stemmers in
+opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball
+were developed by Martin Porter and Richard Boulton.
+The full snowball package is available from
+http://snowball.tartarus.org/
+
+============================================================================
+
+Wordpiece tokenizer
+https://github.com/robrua/easy-bert
+
+The MIT License (MIT)
+
+Copyright (c) 2019 Rob Rua
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+============================================================================
+
+ONNX Runtime
+
+MIT License
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+============================================================================
+
+SLF4J API
+https://github.com/qos-ch/slf4j
+
+MIT License
+
+Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland)
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+============================================================================
+
+jackson-databind
+https://github.com/FasterXML/jackson-databind
+The Apache Software License, Version 2.0
diff --git a/solr/licenses/opennlp-dl-2.5.4.jar.sha1 b/solr/licenses/opennlp-dl-2.5.4.jar.sha1
new file mode 100644
index 00000000000..eda30bfba3c
--- /dev/null
+++ b/solr/licenses/opennlp-dl-2.5.4.jar.sha1
@@ -0,0 +1 @@
+d30a448d1fd548689c576f9ec17e25bf3701e086
diff --git a/solr/licenses/opennlp-tools-2.5.3.jar.sha1 b/solr/licenses/opennlp-tools-2.5.3.jar.sha1
deleted file mode 100644
index fb01299fa29..00000000000
--- a/solr/licenses/opennlp-tools-2.5.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4b544138ec079c1c73dc2c1b928506871c4b1b47
diff --git a/solr/licenses/opennlp-tools-2.5.4.jar.sha1 b/solr/licenses/opennlp-tools-2.5.4.jar.sha1
new file mode 100644
index 00000000000..a721d42ecdd
--- /dev/null
+++ b/solr/licenses/opennlp-tools-2.5.4.jar.sha1
@@ -0,0 +1 @@
+11a7671f2169280c0e24b979cb326833b96ca246
diff --git a/solr/licenses/opennlp-tools-NOTICE.txt b/solr/licenses/opennlp-tools-NOTICE.txt
index 68a08dc4d11..f1d46214b66 100644
--- a/solr/licenses/opennlp-tools-NOTICE.txt
+++ b/solr/licenses/opennlp-tools-NOTICE.txt
@@ -1,6 +1,6 @@
Apache OpenNLP Tools
-Copyright 2015 The Apache Software Foundation
+Copyright 2025 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/solr/modules/analysis-extras/build.gradle b/solr/modules/analysis-extras/build.gradle
index ad1b8714915..f7fa13c8f76 100644
--- a/solr/modules/analysis-extras/build.gradle
+++ b/solr/modules/analysis-extras/build.gradle
@@ -28,11 +28,14 @@ dependencies {
implementation libs.apache.lucene.analysis.icu
runtimeOnly libs.apache.lucene.analysis.morfologik
implementation libs.apache.lucene.analysis.opennlp
+
+ implementation libs.onnxruntime
runtimeOnly libs.apache.lucene.analysis.smartcn
runtimeOnly libs.apache.lucene.analysis.stempel
implementation libs.apache.lucene.core
// NOTE: Need to stay on same version of opennlp-tools as lucene-analysis-opennlp
implementation libs.apache.opennlp.tools
+ implementation libs.apache.opennlp.dl
implementation libs.slf4j.api
testImplementation project(':solr:test-framework')
diff --git a/solr/modules/analysis-extras/gradle.lockfile b/solr/modules/analysis-extras/gradle.lockfile
index ad192ec7893..9ef64b66865 100644
--- a/solr/modules/analysis-extras/gradle.lockfile
+++ b/solr/modules/analysis-extras/gradle.lockfile
@@ -31,6 +31,7 @@ com.ibm.icu:icu4j:77.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLi
com.j256.simplemagic:simplemagic:1.17=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
com.jayway.jsonpath:json-path:2.9.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
com.lmax:disruptor:3.4.4=solrPlatformLibs
+com.microsoft.onnxruntime:onnxruntime:1.20.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
com.tdunning:t-digest:3.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
commons-cli:commons-cli:1.10.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
commons-codec:commons-codec:1.19.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
@@ -113,7 +114,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=apiHelper,jarValidation,runtimeCl
org.apache.lucene:lucene-spatial3d:10.2.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-suggest:10.2.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.opennlp:opennlp-dl:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper-jute:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath
diff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.java
new file mode 100644
index 00000000000..090964d3178
--- /dev/null
+++ b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.java
@@ -0,0 +1,567 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
+import ai.onnxruntime.OrtException;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import opennlp.dl.InferenceOptions;
+import opennlp.dl.doccat.DocumentCategorizerDL;
+import opennlp.dl.doccat.scoring.AverageClassificationScoringStrategy;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.filestore.ClusterFileStore;
+import org.apache.solr.filestore.FileStore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.processor.FieldMutatingUpdateProcessor.FieldNameSelector;
+import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory.SelectorParams;
+import org.apache.solr.util.plugin.SolrCoreAware;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DocumentCategorizerUpdateProcessorFactory extends UpdateRequestProcessorFactory
+ implements SolrCoreAware {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String SOURCE_PARAM = "source";
+ public static final String DEST_PARAM = "dest";
+ public static final String PATTERN_PARAM = "pattern";
+ public static final String REPLACEMENT_PARAM = "replacement";
+ public static final String MODEL_PARAM = "modelFile";
+ public static final String VOCAB_PARAM = "vocabFile";
+
+ private Path solrHome;
+
+ private SelectorParams srcInclusions = new SelectorParams();
+ private Collection<SelectorParams> srcExclusions = new ArrayList<>();
+
+ private FieldNameSelector srcSelector = null;
+
+ private String model = null;
+ private String vocab = null;
+ private String analyzerFieldType = null;
+
+ /**
+ * If pattern is null, this this is a literal field name. If pattern is non-null then this is a
+ * replacement string that may contain meta-characters (ie: capture group identifiers)
+ *
+ * @see #pattern
+ */
+ private String dest = null;
+
+ /**
+ * @see #dest
+ */
+ private Pattern pattern = null;
+
+ protected final FieldNameSelector getSourceSelector() {
+ if (null != srcSelector) return srcSelector;
+
+ throw new SolrException(
+ SERVER_ERROR, "selector was never initialized, inform(SolrCore) never called???");
+ }
+
+ @Override
+ public void init(NamedList<?> args) {
+
+ // high level (loose) check for which type of config we have.
+ //
+ // individual init methods do more strict syntax checking
+ if (0 <= args.indexOf(SOURCE_PARAM, 0) && 0 <= args.indexOf(DEST_PARAM, 0)) {
+ initSourceSelectorSyntax(args);
+ } else if (0 <= args.indexOf(PATTERN_PARAM, 0) && 0 <= args.indexOf(REPLACEMENT_PARAM, 0)) {
+ initSimpleRegexReplacement(args);
+ } else {
+ throw new SolrException(
+ SERVER_ERROR,
+ "A combination of either '"
+ + SOURCE_PARAM
+ + "' + '"
+ + DEST_PARAM
+ + "', or '"
+ + REPLACEMENT_PARAM
+ + "' + '"
+ + PATTERN_PARAM
+ + "' init params are mandatory");
+ }
+
+ Object modelParam = args.remove(MODEL_PARAM);
+ if (null == modelParam) {
+ throw new SolrException(SERVER_ERROR, "Missing required init param '" + MODEL_PARAM + "'");
+ }
+ if (!(modelParam instanceof CharSequence)) {
+ throw new SolrException(SERVER_ERROR, "Init param '" + MODEL_PARAM + "' must be a <str>");
+ }
+ model = modelParam.toString();
+ log.info("In OpenNLP doccat - model: {}", model);
+
+ Object vocabParam = args.remove(VOCAB_PARAM);
+ if (null == vocabParam) {
+ throw new SolrException(SERVER_ERROR, "Missing required init param '" + VOCAB_PARAM + "'");
+ }
+ if (!(vocabParam instanceof CharSequence)) {
+ throw new SolrException(SERVER_ERROR, "Init param '" + VOCAB_PARAM + "' must be a <str>");
+ }
+ vocab = vocabParam.toString();
+
+ if (0 < args.size()) {
+ throw new SolrException(SERVER_ERROR, "Unexpected init param(s): '" + args.getName(0) + "'");
+ }
+
+ super.init(args);
+ }
+
+ /**
+ * init helper method that should only be called when we know for certain that both the "source"
+ * and "dest" init params do <em>not</em> exist.
+ */
+ private void initSimpleRegexReplacement(NamedList<?> args) {
+ // The syntactic sugar for the case where there is only one regex pattern for source and the
+ // same pattern
+ // is used for the destination pattern...
+ //
+ // pattern != null && replacement != null
+ //
+ // ...as top level elements, with no other config options specified
+
+ // if we got here we know we had pattern and replacement, now check for the other two so that
+ // we can give a better
+ // message than "unexpected"
+ if (0 <= args.indexOf(SOURCE_PARAM, 0) || 0 <= args.indexOf(DEST_PARAM, 0)) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Short hand syntax must not be mixed with full syntax. Found "
+ + PATTERN_PARAM
+ + " and "
+ + REPLACEMENT_PARAM
+ + " but also found "
+ + SOURCE_PARAM
+ + " or "
+ + DEST_PARAM);
+ }
+
+ assert args.indexOf(SOURCE_PARAM, 0) < 0;
+
+ Object patt = args.remove(PATTERN_PARAM);
+ Object replacement = args.remove(REPLACEMENT_PARAM);
+
+ if (null == patt || null == replacement) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init params '"
+ + PATTERN_PARAM
+ + "' and '"
+ + REPLACEMENT_PARAM
+ + "' are both mandatory if '"
+ + SOURCE_PARAM
+ + "' and '"
+ + DEST_PARAM
+ + "' are not both specified");
+ }
+
+ if (0 != args.size()) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init params '"
+ + REPLACEMENT_PARAM
+ + "' and '"
+ + PATTERN_PARAM
+ + "' must be children of '"
+ + DEST_PARAM
+ + "' to be combined with other options.");
+ }
+
+ if (!(replacement instanceof String)) {
+ throw new SolrException(
+ SERVER_ERROR, "Init param '" + REPLACEMENT_PARAM + "' must be a string (i.e. <str>)");
+ }
+ if (!(patt instanceof String)) {
+ throw new SolrException(
+ SERVER_ERROR, "Init param '" + PATTERN_PARAM + "' must be a string (i.e. <str>)");
+ }
+
+ dest = replacement.toString();
+ try {
+ this.pattern = Pattern.compile(patt.toString());
+ } catch (PatternSyntaxException pe) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param " + PATTERN_PARAM + " is not a valid regex pattern: " + patt,
+ pe);
+ }
+ srcInclusions = new SelectorParams();
+ srcInclusions.fieldRegex = Collections.singletonList(this.pattern);
+ }
+
+ /**
+ * init helper method that should only be called when we know for certain that both the "source"
+ * and "dest" init params <em>do</em> exist.
+ */
+ private void initSourceSelectorSyntax(NamedList<?> args) {
+ // Full and complete syntax where source and dest are mandatory.
+ //
+ // source may be a single string or a selector.
+ // dest may be a single string or list containing pattern and replacement
+ //
+ // source != null && dest != null
+
+ // if we got here we know we had source and dest, now check for the other two so that we can
+ // give a better
+ // message than "unexpected"
+ if (0 <= args.indexOf(PATTERN_PARAM, 0) || 0 <= args.indexOf(REPLACEMENT_PARAM, 0)) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Short hand syntax must not be mixed with full syntax. Found "
+ + SOURCE_PARAM
+ + " and "
+ + DEST_PARAM
+ + " but also found "
+ + PATTERN_PARAM
+ + " or "
+ + REPLACEMENT_PARAM);
+ }
+
+ Object d = args.remove(DEST_PARAM);
+ assert null != d;
+
+ List<?> sources = args.getAll(SOURCE_PARAM);
+ assert null != sources;
+
+ if (1 == sources.size()) {
+ if (sources.get(0) instanceof NamedList) {
+ // nested set of selector options
+ NamedList<?> selectorConfig = (NamedList<?>) args.remove(SOURCE_PARAM);
+
+ srcInclusions = parseSelectorParams(selectorConfig);
+
+ List<?> excList = selectorConfig.getAll("exclude");
+
+ for (Object excObj : excList) {
+ if (null == excObj) {
+ throw new SolrException(
+ SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' can not be null");
+ }
+ if (!(excObj instanceof NamedList)) {
+ throw new SolrException(
+ SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' must be <lst/>");
+ }
+ NamedList<?> exc = (NamedList<?>) excObj;
+ srcExclusions.add(parseSelectorParams(exc));
+ if (0 < exc.size()) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + SOURCE_PARAM
+ + "' has unexpected 'exclude' sub-param(s): '"
+ + selectorConfig.getName(0)
+ + "'");
+ }
+ // call once per instance
+ selectorConfig.remove("exclude");
+ }
+
+ if (0 < selectorConfig.size()) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + SOURCE_PARAM
+ + "' contains unexpected child param(s): '"
+ + selectorConfig.getName(0)
+ + "'");
+ }
+ // consume from the named list so it doesn't interfere with subsequent processing
+ sources.remove(0);
+ }
+ }
+ if (1 <= sources.size()) {
+ // source better be one or more strings
+ srcInclusions.fieldName = new HashSet<>(args.removeConfigArgs("source"));
+ }
+ if (srcInclusions == null) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init params do not specify any field from which to extract entities, please supply either "
+ + SOURCE_PARAM
+ + " and "
+ + DEST_PARAM
+ + " or "
+ + PATTERN_PARAM
+ + " and "
+ + REPLACEMENT_PARAM
+ + ". See javadocs"
+ + "for OpenNLPExtractNamedEntitiesUpdateProcessor for further details.");
+ }
+
+ if (d instanceof NamedList) {
+ NamedList<?> destList = (NamedList<?>) d;
+
+ Object patt = destList.remove(PATTERN_PARAM);
+ Object replacement = destList.remove(REPLACEMENT_PARAM);
+
+ if (null == patt || null == replacement) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + DEST_PARAM
+ + "' children '"
+ + PATTERN_PARAM
+ + "' and '"
+ + REPLACEMENT_PARAM
+ + "' are both mandatory and can not be null");
+ }
+ if (!(patt instanceof String && replacement instanceof String)) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + DEST_PARAM
+ + "' children '"
+ + PATTERN_PARAM
+ + "' and '"
+ + REPLACEMENT_PARAM
+ + "' must both be strings (i.e. <str>)");
+ }
+ if (0 != destList.size()) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + DEST_PARAM
+ + "' has unexpected children: '"
+ + destList.getName(0)
+ + "'");
+ }
+
+ try {
+ this.pattern = Pattern.compile(patt.toString());
+ } catch (PatternSyntaxException pe) {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + DEST_PARAM
+ + "' child '"
+ + PATTERN_PARAM
+ + " is not a valid regex pattern: "
+ + patt,
+ pe);
+ }
+ dest = replacement.toString();
+
+ } else if (d instanceof String) {
+ dest = d.toString();
+ } else {
+ throw new SolrException(
+ SERVER_ERROR,
+ "Init param '"
+ + DEST_PARAM
+ + "' must either be a string "
+ + "(i.e. <str>) or a list (i.e. <lst>) containing '"
+ + PATTERN_PARAM
+ + "' and '"
+ + REPLACEMENT_PARAM);
+ }
+ }
+
+ @Override
+ public void inform(final SolrCore core) {
+ this.solrHome = core.getCoreContainer().getSolrHome();
+ srcSelector =
+ FieldMutatingUpdateProcessor.createFieldNameSelector(
+ core.getResourceLoader(),
+ core,
+ srcInclusions,
+ FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);
+
+ for (SelectorParams exc : srcExclusions) {
+ srcSelector =
+ FieldMutatingUpdateProcessor.wrap(
+ srcSelector,
+ FieldMutatingUpdateProcessor.createFieldNameSelector(
+ core.getResourceLoader(),
+ core,
+ exc,
+ FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));
+ }
+ }
+
+ @Override
+ public final UpdateRequestProcessor getInstance(
+ SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ final FieldNameSelector srcSelector = getSourceSelector();
+ return new UpdateRequestProcessor(next) {
+
+ DocumentCategorizerDL documentCategorizerDL = null;
+
+ {
+ // Initialize the categorizer.
+ FileStore fs = req.getCoreContainer().getFileStore();
+
+ var path = solrHome.resolve(ClusterFileStore.FILESTORE_DIRECTORY);
+ Path modelFile = Path.of(model);
+ Path vocabFile = Path.of(vocab);
+
+ if (!Files.exists(modelFile)) {
+ modelFile = path.resolve(model);
+ }
+ if (!Files.exists(vocabFile)) {
+ vocabFile = path.resolve(vocab);
+ }
+
+ log.info("initializing the documentCategorizerDL");
+ try {
+ documentCategorizerDL =
+ new DocumentCategorizerDL(
+ modelFile.toFile(),
+ vocabFile.toFile(),
+ getCategories(),
+ new AverageClassificationScoringStrategy(),
+ new InferenceOptions());
+ } catch (IOException e) {
+ log.warn("Attempted to initialize documentCategorizerDL", e);
+ } catch (OrtException e) {
+ log.warn("Attempted to initialize documentCategorizerDL", e);
+ }
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+
+ final SolrInputDocument doc = cmd.getSolrInputDocument();
+
+ // Destination may be regex replace string, or "{EntityType}" replaced by
+ // each entity's type, both of which can cause multiple output fields.
+ Map<String, SolrInputField> destMap = new HashMap<>();
+
+ // preserve initial values
+ for (final String fname : doc.getFieldNames()) {
+ if (!srcSelector.shouldMutate(fname)) continue;
+
+ Collection<Object> srcFieldValues = doc.getFieldValues(fname);
+ if (srcFieldValues == null || srcFieldValues.isEmpty()) continue;
+
+ String resolvedDest = dest;
+
+ if (pattern != null) {
+ Matcher matcher = pattern.matcher(fname);
+ if (matcher.find()) {
+ resolvedDest = matcher.replaceAll(dest);
+ } else {
+ log.debug(
+ "srcSelector.shouldMutate('{}') returned true, "
+ + "but replacement pattern did not match, field skipped.",
+ fname);
+ continue;
+ }
+ }
+
+ for (Object val : srcFieldValues) {
+ for (Pair<String, String> entity : classify(val)) {
+ SolrInputField destField = null;
+ // String classification = entity.first();
+ String classificationValue = entity.second();
+ final String resolved = resolvedDest;
+ if (doc.containsKey(resolved)) {
+ destField = doc.getField(resolved);
+ } else {
+ SolrInputField targetField = destMap.get(resolved);
+ if (targetField == null) {
+ destField = new SolrInputField(resolved);
+ } else {
+ destField = targetField;
+ }
+ }
+ destField.addValue(classificationValue);
+
+ // put it in map to avoid concurrent modification...
+ destMap.put(resolved, destField);
+ }
+ }
+ }
+
+ for (Map.Entry<String, SolrInputField> entry : destMap.entrySet()) {
+ doc.put(entry.getKey(), entry.getValue());
+ }
+ super.processAdd(cmd);
+ }
+
+ private List<Pair<String, String>> classify(Object srcFieldValue) {
+
+ String fullText = srcFieldValue.toString();
+
+ // Send the fullText to the model for classification.
+ log.info("calling categorizer()");
+ final double[] result = documentCategorizerDL.categorize(new String[] {fullText});
+
+ // Add the categories to the list and return it.
+ // Just take the top category for now.
+ // TODO: Allow for a threshold value for returning categories.
+
+ List<Pair<String, String>> classifications = new ArrayList<>();
+
+ String bestCategory = documentCategorizerDL.getBestCategory(result);
+ if (log.isInfoEnabled()) {
+ log.info("best category = {}", bestCategory);
+ }
+
+ Pair<String, String> pair = new Pair<>("classification", bestCategory);
+ classifications.add(pair);
+
+ return classifications;
+ }
+ };
+ }
+
+ /** macro */
+ private static SelectorParams parseSelectorParams(NamedList<?> args) {
+ return FieldMutatingUpdateProcessorFactory.parseSelectorParams(args);
+ }
+
+ private Map<Integer, String> getCategories() {
+
+ // TODO: Read these from the Solr config for the processor.
+
+ final Map<Integer, String> categories = new HashMap<>();
+
+ categories.put(0, "very bad");
+ categories.put(1, "bad");
+ categories.put(2, "neutral");
+ categories.put(3, "good");
+ categories.put(4, "very good");
+
+ return categories;
+ }
+}
diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.bin b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.bin
new file mode 100644
index 00000000000..1e489fb4ffe
--- /dev/null
+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.bin
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MOCK_MODEL_FILE_FOR_DOCUMENT_CATEGORIZER_TESTS
+This is a placeholder file and not an actual model file.
+In a real test environment, this would be replaced with a proper model file
+or the tests would be modified to mock the file loading operations.
diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txt b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txt
new file mode 100644
index 00000000000..508ced5ee90
--- /dev/null
+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txt
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Mock vocabulary file for DocumentCategorizer tests
+good
+bad
+excellent
+terrible
+fine
+average
+neutral
+very
+love
+product
+recommend
+quality
+service
+broke
+immediately
+special
+performance
+acceptable
diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xml
new file mode 100644
index 00000000000..4aa89d15528
--- /dev/null
+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="document-categorizer" version="1.6">
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true"/>
+ <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true"/>
+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <!-- For document categorization -->
+ <fieldType name="categorizer_text" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <!--filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/-->
+ </analyzer>
+ </fieldType>
+
+ <!-- Fields used in tests -->
+ <field name="id" type="string" indexed="true" stored="true" required="true"/>
+ <field name="text_s" type="string" indexed="true" stored="true"/>
+ <field name="text1_s" type="string" indexed="true" stored="true"/>
+ <field name="text2_s" type="string" indexed="true" stored="true"/>
+ <field name="text0_s" type="string" indexed="true" stored="true"/>
+ <field name="ignore_s" type="string" indexed="true" stored="true"/>
+ <field name="review_text_s" type="string" indexed="true" stored="true"/>
+ <field name="comments_s" type="string" indexed="true" stored="true"/>
+ <field name="review_1_text" type="string" indexed="true" stored="true"/>
+ <field name="review_2_text" type="string" indexed="true" stored="true"/>
+
+ <!-- Destination fields -->
+ <field name="category_s" type="strings" indexed="true" stored="true" multiValued="true"/>
+ <field name="result_1_category" type="string" indexed="true" stored="true"/>
+ <field name="result_2_category" type="string" indexed="true" stored="true"/>
+
+ <uniqueKey>id</uniqueKey>
+</schema>
diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xml
new file mode 100644
index 00000000000..c40f3e17f3c
--- /dev/null
+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+ <requestHandler name="/select" class="solr.SearchHandler"></requestHandler>
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <updateRequestProcessorChain name="categorize-single">
+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">
+ <str name="modelFile">doccat-model.bin</str>
+ <str name="vocabFile">doccat-vocab.txt</str>
+ <str name="source">text_s</str>
+ <str name="dest">category_s</str>
+ </processor>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="categorize-multi">
+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">
+ <str name="modelFile">doccat-model.bin</str>
+ <str name="vocabFile">doccat-vocab.txt</str>
+ <str name="source">text1_s</str>
+ <str name="source">text2_s</str>
+ <str name="dest">category_s</str>
+ </processor>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="categorize-regex">
+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">
+ <str name="modelFile">doccat-model.bin</str>
+ <str name="vocabFile">doccat-vocab.txt</str>
+ <lst name="source">
+ <str name="fieldRegex">.*_text_s|comments_s</str>
+ </lst>
+ <str name="dest">category_s</str>
+ </processor>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="categorize-field-regex">
+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">
+ <str name="modelFile">doccat-model.bin</str>
+ <str name="vocabFile">doccat-vocab.txt</str>
+ <lst name="source">
+ <str name="fieldRegex">review_\d_text</str>
+ </lst>
+ <lst name="dest">
+ <str name="pattern">review_(\d)_text</str>
+ <str name="replacement">result_$1_category</str>
+ </lst>
+ </processor>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="categorize-selector">
+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">
+ <str name="modelFile">doccat-model.bin</str>
+ <str name="vocabFile">doccat-vocab.txt</str>
+ <lst name="source">
+ <str name="fieldRegex">text\d_s</str>
+ <lst name="exclude">
+ <str name="fieldRegex">text0_s</str>
+ </lst>
+ </lst>
+ <str name="dest">category_s</str>
+ </processor>
+ </updateRequestProcessorChain>
+</config>
\ No newline at end of file
diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.java
new file mode 100644
index 00000000000..74af837305c
--- /dev/null
+++ b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.nio.file.Path;
+import org.apache.commons.io.file.PathUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests for {@link DocumentCategorizerUpdateProcessorFactory}
+ *
+ * <p>Focused on verifying how DocumentCategorizerUpdateProcessorFactory is set up to run.
+ *
+ * <p>See test_opennlp.bats for a complete system test of DocumentCategorizerUpdateProcessorFactory.
+ */
+public class TestConfiguringDocumentCategorizerUpdateProcessorFactory
+ extends UpdateProcessorTestBase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ Path testHome = createTempDir();
+ PathUtils.copyDirectory(getFile("analysis-extras/solr"), testHome);
+ initCore("solrconfig-document-categorizer.xml", "schema-document-categorizer.xml", testHome);
+ }
+
+ @Test
+ public void testBasicConfiguration() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Add source and dest
+ args.add("source", "text_field");
+ args.add("dest", "category_field");
+
+ // Add required model files
+ args.add("modelFile", "doccat-model.bin");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ // Should initialize without errors
+ factory.init(args);
+ }
+
+ @Test(expected = SolrException.class)
+ public void testMissingModelFile() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Add source and dest but miss modelFile
+ args.add("source", "text_field");
+ args.add("dest", "category_field");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ factory.init(args);
+ }
+
+ @Test(expected = SolrException.class)
+ public void testMissingVocabFile() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Add source and dest but miss vocabFile
+ args.add("source", "text_field");
+ args.add("dest", "category_field");
+ args.add("modelFile", "doccat-model.bin");
+
+ factory.init(args);
+ }
+
+ @Test(expected = SolrException.class)
+ public void testMissingSourceAndDest() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Add only model files but missing source/dest
+ args.add("modelFile", "doccat-model.bin");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ factory.init(args);
+ }
+
+ @Test
+ public void testRegexConfiguration() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Add source parameter
+ NamedList<Object> sourceArgs = new NamedList<>();
+ sourceArgs.add("fieldRegex", "text_.*");
+ args.add("source", sourceArgs);
+
+ // Create pattern and replacement as part of a dest structure
+ NamedList<Object> destArgs = new NamedList<>();
+ destArgs.add("pattern", "text_(.*)");
+ destArgs.add("replacement", "category_$1");
+ args.add("dest", destArgs);
+
+ // Add required model files
+ args.add("modelFile", "doccat-model.bin");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ // Should initialize without errors
+ factory.init(args);
+ }
+
+ @Test
+ public void testSourceSelector() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Create a complex source selector
+ NamedList<Object> sourceArgs = new NamedList<>();
+ sourceArgs.add("fieldRegex", "text_.*");
+ NamedList<Object> excludeArgs = new NamedList<>();
+ excludeArgs.add("fieldRegex", "text_ignore_.*");
+ sourceArgs.add("exclude", excludeArgs);
+
+ args.add("source", sourceArgs);
+ args.add("dest", "category_field");
+ args.add("modelFile", "doccat-model.bin");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ // Should initialize without errors
+ factory.init(args);
+ }
+
+ @Test
+ public void testDestWithPatternReplacement() {
+ DocumentCategorizerUpdateProcessorFactory factory =
+ new DocumentCategorizerUpdateProcessorFactory();
+ NamedList<Object> args = new NamedList<>();
+
+ // Simple source
+ args.add("source", "text_field");
+
+ // Dest with pattern and replacement
+ NamedList<Object> destArgs = new NamedList<>();
+ destArgs.add("pattern", "text_(.*)");
+ destArgs.add("replacement", "category_$1");
+ args.add("dest", destArgs);
+
+ // Add required model files
+ args.add("modelFile", "doccat-model.bin");
+ args.add("vocabFile", "doccat-vocab.txt");
+
+ // Should initialize without errors
+ factory.init(args);
+ }
+}
diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfile
index a42495189a3..0aec085dd80 100644
--- a/solr/modules/extraction/gradle.lockfile
+++ b/solr/modules/extraction/gradle.lockfile
@@ -146,7 +146,7 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ru
org.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.pdfbox:fontbox:2.0.26=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.pdfbox:jbig2-imageio:3.0.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.pdfbox:jempbox:1.8.16=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
diff --git a/solr/modules/langid/gradle.lockfile b/solr/modules/langid/gradle.lockfile
index 09899293f8a..7b56fcac95b 100644
--- a/solr/modules/langid/gradle.lockfile
+++ b/solr/modules/langid/gradle.lockfile
@@ -108,7 +108,7 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ru
org.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.tika:tika-core:1.28.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper-jute:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
diff --git a/solr/modules/sql/gradle.lockfile b/solr/modules/sql/gradle.lockfile
index 89e85fd3495..713148901f9 100644
--- a/solr/modules/sql/gradle.lockfile
+++ b/solr/modules/sql/gradle.lockfile
@@ -32,6 +32,7 @@ com.ibm.icu:icu4j:77.1=jarValidation,testRuntimeClasspath
com.j256.simplemagic:simplemagic:1.17=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
com.jayway.jsonpath:json-path:2.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
com.lmax:disruptor:3.4.4=solrPlatformLibs
+com.microsoft.onnxruntime:onnxruntime:1.20.0=jarValidation,testRuntimeClasspath
com.tdunning:t-digest:3.3=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
commons-cli:commons-cli:1.10.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
commons-codec:commons-codec:1.19.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
@@ -121,7 +122,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ru
org.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath
org.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.opennlp:opennlp-tools:2.5.3=jarValidation,testRuntimeClasspath
+org.apache.opennlp:opennlp-dl:2.5.4=jarValidation,testRuntimeClasspath
+org.apache.opennlp:opennlp-tools:2.5.4=jarValidation,testRuntimeClasspath
org.apache.zookeeper:zookeeper-jute:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
org.apache.zookeeper:zookeeper:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath
org.apiguardian:apiguardian-api:1.1.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
diff --git a/solr/packaging/test/test_opennlp.bats b/solr/packaging/test/test_opennlp.bats
new file mode 100644
index 00000000000..c2fd6755b38
--- /dev/null
+++ b/solr/packaging/test/test_opennlp.bats
@@ -0,0 +1,119 @@
+#!/usr/bin/env bats
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load bats_helper
+
+setup_file() {
+ common_clean_setup
+
+}
+
+teardown_file() {
+ common_setup
+ solr stop --all
+}
+
+setup() {
+ common_setup
+}
+
+teardown() {
+ # save a snapshot of SOLR_HOME for failed tests
+ save_home_on_failure
+}
+
+# This BATS style test is really just to help explore the space of Modern NLP in
+# Apache Solr, versus a "true" integration test that I want to have run regularly.
+# On the other hand, since integrating NLP requires a lot of steps, maybe having this
+# long test as an "integration" test is something we decide is okay?
+# I also have dreams of incorporating this as code snippets in a Tutorial via the ascii doc tags
+# like we use for the SolrJ code snippets. That way we know the snippets continue to work!
+@test "Check lifecycle of sentiment classification" {
+
+ echo "Downloading onnx model and vocab..."
+
+ mkdir -p ${SOLR_TIP}/models/sentiment/
+
+ # Models originally sourced from Huggingface and store on ASF hardware
+ wget -O ${SOLR_TIP}/models/sentiment/model.onnx https://nightlies.apache.org/solr/opennlp-dl-integration/model_quantized.onnx
+ wget -O ${SOLR_TIP}/models/sentiment/vocab.txt https://nightlies.apache.org/solr/opennlp-dl-integration/vocab.txt
+ # wget -O ${SOLR_TIP}/models/sentiment/model.onnx https://huggingface.co/onnx-community/bert-base-multilingual-uncased-sentiment-ONNX/resolve/main/onnx/model_quantized.onnx
+ # wget -O ${SOLR_TIP}/models/sentiment/vocab.txt https://huggingface.co/onnx-community/bert-base-multilingual-uncased-sentiment-ONNX/raw/main/vocab.txt
+
+ run ls -alh ${SOLR_TIP}/modules/analysis-extras/lib
+ refute_output --partial "onnxruntime_gpu"
+
+ # Can't figure out magic policy stuff to allow loading ONNX, so disable security manager.
+ export SOLR_SECURITY_MANAGER_ENABLED=false
+
+ solr start -m 4g -Dsolr.modules=analysis-extras -Denable.packages=true
+ solr assert --started http://localhost:${SOLR_PORT}/solr --timeout 5000
+
+ run solr create -c COLL_NAME
+ assert_output --partial "Created collection 'COLL_NAME'"
+
+ curl -X POST -H 'Content-type:application/json' --data-binary '{
+ "add-field":{
+ "name":"name",
+ "type":"string",
+ "stored":true }
+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/schema"
+
+ curl -X POST -H 'Content-type:application/json' --data-binary '{
+ "add-field":{
+ "name":"name_sentiment",
+ "type":"string",
+ "stored":true }
+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/schema"
+
+ run curl --data-binary @${SOLR_TIP}/models/sentiment/vocab.txt -X PUT "http://localhost:${SOLR_PORT}/api/cluster/filestore/files/models/sentiment/vocab.txt"
+ assert_output --partial '"status":0'
+
+ run curl --data-binary @${SOLR_TIP}/models/sentiment/model.onnx -X PUT "http://localhost:${SOLR_PORT}/api/cluster/filestore/files/models/sentiment/model.onnx"
+ assert_output --partial '"status":0'
+
+ run curl -X POST -H 'Content-type:application/json' -d '{
+ "add-updateprocessor": {
+ "name": "sentimentClassifier",
+ "class": "solr.processor.DocumentCategorizerUpdateProcessorFactory",
+ "modelFile": "models/sentiment/model.onnx",
+ "vocabFile": "models/sentiment/vocab.txt",
+ "source": "name",
+ "dest": "name_sentiment"
+ }
+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/config"
+ assert_output --partial '"status":0'
+
+ run curl -X POST -H 'Content-type:application/json' -d '[
+ {
+ "id":"good",
+ "name" : "that was an awesome movie!"
+ },
+ {
+ "id":"bad",
+ "name" : "that movie was bad and terrible"
+ }
+ ]' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/update/json?processor=sentimentClassifier&commit=true"
+
+ assert_output --partial '"status":0'
+
+ run curl -X GET "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=id:good"
+ assert_output --partial '"name_sentiment":"very good"'
+
+ run curl -X GET "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=id:bad"
+ assert_output --partial '"name_sentiment":"very bad"'
+}
diff --git a/solr/solr-ref-guide/antora.yml b/solr/solr-ref-guide/antora.yml
index 2882d5f4655..d612229ed36 100644
--- a/solr/solr-ref-guide/antora.yml
+++ b/solr/solr-ref-guide/antora.yml
@@ -48,7 +48,7 @@ asciidoc:
dep-version-commons-codec: '1.15'
dep-version-dropwizard: '4.2.12'
dep-version-log4j: '2.19.0'
- dep-version-opennlp: '1.9.4'
+ dep-version-opennlp: '2.5.4'
dep-version-tika: '1.28.4'
dep-version-zookeeper: '3.8.0'
dep-version-lucene: '9.4.2'