Apache Solr open-source search software
SOLR-17023: Use Modern NLP Models via ONNX and Apache OpenNLP with Solr (#1999)
Eric Pugh08c07309d878
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.tomlindex 6c2d83df338..e9760f446cc 100644--- a/gradle/libs.versions.toml+++ b/gradle/libs.versions.toml@@ -38,7 +38,7 @@ apache-httpcomponents-httpmime = "4.5.14"apache-kafka = "3.9.0"apache-log4j = "2.21.0"apache-lucene = "10.2.1"-apache-opennlp = "2.5.3"+apache-opennlp = "2.5.4"apache-poi = "5.2.2"apache-rat = "0.15"apache-tika = "1.28.5"@@ -171,6 +171,7 @@ nlopez-compose = "0.4.27"nodegradle-node = "7.1.0"# @keep Node JS version used in node.gradle (LTS)nodejs = "22.18.0"+onnx = "1.20.0"openapi = "7.14.0"openjdk-jmh = "1.37"opentelemetry = "1.53.0"@@ -285,6 +286,7 @@ apache-lucene-sandbox = { module = "org.apache.lucene:lucene-sandbox", version.rapache-lucene-spatialextras = { module = "org.apache.lucene:lucene-spatial-extras", version.ref = "apache-lucene" }apache-lucene-suggest = { module = "org.apache.lucene:lucene-suggest", version.ref = "apache-lucene" }apache-lucene-testframework = { module = "org.apache.lucene:lucene-test-framework", version.ref = "apache-lucene" }+apache-opennlp-dl = { module = "org.apache.opennlp:opennlp-dl", version.ref = "apache-opennlp" }apache-opennlp-tools = { module = "org.apache.opennlp:opennlp-tools", version.ref = "apache-opennlp" }apache-poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "apache-poi" }apache-poi-poi = { module = "org.apache.poi:poi", version.ref = "apache-poi" }@@ -473,6 +475,7 @@ netty-transport-classes-epoll = { module = "io.netty:netty-transport-classes-eponetty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" }nimbusds-josejwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbusds-josejwt" }nlopez-compose-ktlintrules = { module = "io.nlopez.compose.rules:ktlint", version.ref = "nlopez-compose" }+onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnx" }openjdk-jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "openjdk-jmh" }openjdk-jmh-generatorannprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "openjdk-jmh" }opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" }diff --git a/solr/core/gradle.lockfile b/solr/core/gradle.lockfileindex 0079eb41880..6cd8dff00f6 100644--- a/solr/core/gradle.lockfile+++ b/solr/core/gradle.lockfile@@ -33,6 +33,7 @@ com.google.protobuf:protobuf-java:3.25.8=annotationProcessor,errorprone,testAnnocom.ibm.icu:icu4j:77.1=jarValidation,testRuntimeClasspathcom.j256.simplemagic:simplemagic:1.17=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathcom.jayway.jsonpath:json-path:2.9.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath+com.microsoft.onnxruntime:onnxruntime:1.20.0=jarValidation,testRuntimeClasspathcom.tdunning:t-digest:3.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathcommons-cli:commons-cli:1.10.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathcommons-codec:commons-codec:1.19.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath@@ -114,7 +115,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=compileClasspath,jarValidation,ruorg.apache.lucene:lucene-spatial3d:10.2.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.lucene:lucene-suggest:10.2.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath-org.apache.opennlp:opennlp-tools:2.5.3=jarValidation,testRuntimeClasspath+org.apache.opennlp:opennlp-dl:2.5.4=jarValidation,testRuntimeClasspath+org.apache.opennlp:opennlp-tools:2.5.4=jarValidation,testRuntimeClasspathorg.apache.zookeeper:zookeeper-jute:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apiguardian:apiguardian-api:1.1.2=jarValidation,testCompileClasspath,testRuntimeClasspathdiff --git a/solr/licenses/onnxruntime-1.20.0.jar.sha1 b/solr/licenses/onnxruntime-1.20.0.jar.sha1new file mode 100644index 00000000000..a1973ae8b1c--- /dev/null+++ b/solr/licenses/onnxruntime-1.20.0.jar.sha1@@ -0,0 +1 @@+717cd187f7d5080a8cc5c8aa761cafe2db028de1diff --git a/solr/licenses/onnxruntime-LICENSE-MIT.txt b/solr/licenses/onnxruntime-LICENSE-MIT.txtnew file mode 100644index 00000000000..48bc6bb4996--- /dev/null+++ b/solr/licenses/onnxruntime-LICENSE-MIT.txt@@ -0,0 +1,21 @@+MIT License++Copyright (c) Microsoft Corporation++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/solr/licenses/opennlp-LICENSE-ASL.txt b/solr/licenses/opennlp-LICENSE-ASL.txtnew file mode 100644index 00000000000..d1f4d5cc086--- /dev/null+++ b/solr/licenses/opennlp-LICENSE-ASL.txt@@ -0,0 +1,314 @@++ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.++EXTERNAL COMPONENTS++Apache PDFBox includes a number of components with separate copyright notices+and license terms. Your use of these components is subject to the terms and+conditions of the following licenses.++Contributions made to the original PDFBox project:++ Copyright (c) 2002-2007, www.pdfbox.org+ All rights reserved.++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions are met:++ 1. Redistributions of source code must retain the above copyright notice,+ this list of conditions and the following disclaimer.++ 2. Redistributions in binary form must reproduce the above copyright+ notice, this list of conditions and the following disclaimer in the+ documentation and/or other materials provided with the distribution.++ 3. Neither the name of pdfbox; nor the names of its contributors may be+ used to endorse or promote products derived from this software without+ specific prior written permission.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF+ SUCH DAMAGE.++Adobe Font Metrics (AFM) for PDF Core 14 Fonts++ This file and the 14 PostScript(R) AFM files it accompanies may be used,+ copied, and distributed for any purpose and without charge, with or without+ modification, provided that all copyright notices are retained; that the+ AFM files are not distributed without this file; that all modifications+ to this file or any of the AFM files are prominently noted in the modified+ file(s); and that this paragraph is not modified. Adobe Systems has no+ responsibility or obligation to support the use of the AFM files.++CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads)++ Copyright 1990-2009 Adobe Systems Incorporated.+ All rights reserved.++ Redistribution and use in source and binary forms, with or without+ modification, are permitted provided that the following conditions+ are met:++ Redistributions of source code must retain the above copyright notice,+ this list of conditions and the following disclaimer.++ Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++ Neither the name of Adobe Systems Incorporated nor the names of its+ contributors may be used to endorse or promote products derived from this+ software without specific prior written permission.++ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF+ THE POSSIBILITY OF SUCH DAMAGE.++Glyphlist (http://www.adobe.com/devnet/opentype/archives/glyph.html)++ Copyright (c) 1997,1998,2002,2007 Adobe Systems Incorporated++ Permission is hereby granted, free of charge, to any person obtaining a+ copy of this documentation file to use, copy, publish, distribute,+ sublicense, and/or sell copies of the documentation, and to permit+ others to do the same, provided that:+ - No modification, editing or other alteration of this document is+ allowed; and+ - The above copyright notice and this permission notice shall be+ included in all copies of the documentation.++ Permission is hereby granted, free of charge, to any person obtaining a+ copy of this documentation file, to create their own derivative works+ from the content of this document to use, copy, publish, distribute,+ sublicense, and/or sell the derivative works, and to permit others to do+ the same, provided that the derived work is not represented as being a+ copy or version of this document.++ Adobe shall not be liable to any party for any loss of revenue or profit+ or for indirect, incidental, special, consequential, or other similar+ damages, whether based on tort (including without limitation negligence+ or strict liability), contract or other legal or equitable grounds even+ if Adobe has been advised or had reason to know of the possibility of+ such damages. The Adobe materials are provided on an "AS IS" basis.+ Adobe specifically disclaims all express, statutory, or implied+ warranties relating to the Adobe materials, including but not limited to+ those concerning merchantability or fitness for a particular purpose or+ non-infringement of any third party rights regarding the Adobe+ materials.+diff --git a/solr/licenses/opennlp-NOTICE.txt b/solr/licenses/opennlp-NOTICE.txtnew file mode 100644index 00000000000..617c7d9b106--- /dev/null+++ b/solr/licenses/opennlp-NOTICE.txt@@ -0,0 +1,101 @@+Apache OpenNLP+Copyright 2021-2025 The Apache Software Foundation++This product includes software developed at+The Apache Software Foundation (http://www.apache.org/).++============================================================================++The snowball stemmers in+opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball+were developed by Martin Porter and Richard Boulton.+The full snowball package is available from+http://snowball.tartarus.org/++============================================================================++Wordpiece tokenizer+https://github.com/robrua/easy-bert++The MIT License (MIT)++Copyright (c) 2019 Rob Rua++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.++============================================================================++ONNX Runtime++MIT License++Copyright (c) Microsoft Corporation++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.++============================================================================++SLF4J API+https://github.com/qos-ch/slf4j++MIT License++Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland)+All rights reserved.++Permission is hereby granted, free of charge, to any person obtaining+a copy of this software and associated documentation files (the+"Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish,+distribute, sublicense, and/or sell copies of the Software, and to+permit persons to whom the Software is furnished to do so, subject to+the following conditions:++The above copyright notice and this permission notice shall be+included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.++============================================================================++jackson-databind+https://github.com/FasterXML/jackson-databind+The Apache Software License, Version 2.0diff --git a/solr/licenses/opennlp-dl-2.5.4.jar.sha1 b/solr/licenses/opennlp-dl-2.5.4.jar.sha1new file mode 100644index 00000000000..eda30bfba3c--- /dev/null+++ b/solr/licenses/opennlp-dl-2.5.4.jar.sha1@@ -0,0 +1 @@+d30a448d1fd548689c576f9ec17e25bf3701e086diff --git a/solr/licenses/opennlp-tools-2.5.3.jar.sha1 b/solr/licenses/opennlp-tools-2.5.3.jar.sha1deleted file mode 100644index fb01299fa29..00000000000--- a/solr/licenses/opennlp-tools-2.5.3.jar.sha1+++ /dev/null@@ -1 +0,0 @@-4b544138ec079c1c73dc2c1b928506871c4b1b47diff --git a/solr/licenses/opennlp-tools-2.5.4.jar.sha1 b/solr/licenses/opennlp-tools-2.5.4.jar.sha1new file mode 100644index 00000000000..a721d42ecdd--- /dev/null+++ b/solr/licenses/opennlp-tools-2.5.4.jar.sha1@@ -0,0 +1 @@+11a7671f2169280c0e24b979cb326833b96ca246diff --git a/solr/licenses/opennlp-tools-NOTICE.txt b/solr/licenses/opennlp-tools-NOTICE.txtindex 68a08dc4d11..f1d46214b66 100644--- a/solr/licenses/opennlp-tools-NOTICE.txt+++ b/solr/licenses/opennlp-tools-NOTICE.txt@@ -1,6 +1,6 @@Apache OpenNLP Tools-Copyright 2015 The Apache Software Foundation+Copyright 2025 The Apache Software FoundationThis product includes software developed atThe Apache Software Foundation (http://www.apache.org/).diff --git a/solr/modules/analysis-extras/build.gradle b/solr/modules/analysis-extras/build.gradleindex ad1b8714915..f7fa13c8f76 100644--- a/solr/modules/analysis-extras/build.gradle+++ b/solr/modules/analysis-extras/build.gradle@@ -28,11 +28,14 @@ dependencies {implementation libs.apache.lucene.analysis.icuruntimeOnly libs.apache.lucene.analysis.morfologikimplementation libs.apache.lucene.analysis.opennlp++ implementation libs.onnxruntimeruntimeOnly libs.apache.lucene.analysis.smartcnruntimeOnly libs.apache.lucene.analysis.stempelimplementation libs.apache.lucene.core// NOTE: Need to stay on same version of opennlp-tools as lucene-analysis-opennlpimplementation libs.apache.opennlp.tools+ implementation libs.apache.opennlp.dlimplementation libs.slf4j.apitestImplementation project(':solr:test-framework')diff --git a/solr/modules/analysis-extras/gradle.lockfile b/solr/modules/analysis-extras/gradle.lockfileindex ad192ec7893..9ef64b66865 100644--- a/solr/modules/analysis-extras/gradle.lockfile+++ b/solr/modules/analysis-extras/gradle.lockfile@@ -31,6 +31,7 @@ com.ibm.icu:icu4j:77.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLicom.j256.simplemagic:simplemagic:1.17=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcom.jayway.jsonpath:json-path:2.9.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcom.lmax:disruptor:3.4.4=solrPlatformLibs+com.microsoft.onnxruntime:onnxruntime:1.20.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathcom.tdunning:t-digest:3.3=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcommons-cli:commons-cli:1.10.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcommons-codec:commons-codec:1.19.0=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath@@ -113,7 +114,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=apiHelper,jarValidation,runtimeClorg.apache.lucene:lucene-spatial3d:10.2.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-suggest:10.2.1=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath+org.apache.opennlp:opennlp-dl:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper-jute:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper:3.9.4=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathorg.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspathdiff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.javanew file mode 100644index 00000000000..090964d3178--- /dev/null+++ b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/DocumentCategorizerUpdateProcessorFactory.java@@ -0,0 +1,567 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.update.processor;++import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;++import ai.onnxruntime.OrtException;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Collections;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import java.util.regex.PatternSyntaxException;+import opennlp.dl.InferenceOptions;+import opennlp.dl.doccat.DocumentCategorizerDL;+import opennlp.dl.doccat.scoring.AverageClassificationScoringStrategy;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.SolrInputField;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Pair;+import org.apache.solr.core.SolrCore;+import org.apache.solr.filestore.ClusterFileStore;+import org.apache.solr.filestore.FileStore;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.update.AddUpdateCommand;+import org.apache.solr.update.processor.FieldMutatingUpdateProcessor.FieldNameSelector;+import org.apache.solr.update.processor.FieldMutatingUpdateProcessorFactory.SelectorParams;+import org.apache.solr.util.plugin.SolrCoreAware;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class DocumentCategorizerUpdateProcessorFactory extends UpdateRequestProcessorFactory+ implements SolrCoreAware {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public static final String SOURCE_PARAM = "source";+ public static final String DEST_PARAM = "dest";+ public static final String PATTERN_PARAM = "pattern";+ public static final String REPLACEMENT_PARAM = "replacement";+ public static final String MODEL_PARAM = "modelFile";+ public static final String VOCAB_PARAM = "vocabFile";++ private Path solrHome;++ private SelectorParams srcInclusions = new SelectorParams();+ private Collection<SelectorParams> srcExclusions = new ArrayList<>();++ private FieldNameSelector srcSelector = null;++ private String model = null;+ private String vocab = null;+ private String analyzerFieldType = null;++ /**+ * If pattern is null, this this is a literal field name. If pattern is non-null then this is a+ * replacement string that may contain meta-characters (ie: capture group identifiers)+ *+ * @see #pattern+ */+ private String dest = null;++ /**+ * @see #dest+ */+ private Pattern pattern = null;++ protected final FieldNameSelector getSourceSelector() {+ if (null != srcSelector) return srcSelector;++ throw new SolrException(+ SERVER_ERROR, "selector was never initialized, inform(SolrCore) never called???");+ }++ @Override+ public void init(NamedList<?> args) {++ // high level (loose) check for which type of config we have.+ //+ // individual init methods do more strict syntax checking+ if (0 <= args.indexOf(SOURCE_PARAM, 0) && 0 <= args.indexOf(DEST_PARAM, 0)) {+ initSourceSelectorSyntax(args);+ } else if (0 <= args.indexOf(PATTERN_PARAM, 0) && 0 <= args.indexOf(REPLACEMENT_PARAM, 0)) {+ initSimpleRegexReplacement(args);+ } else {+ throw new SolrException(+ SERVER_ERROR,+ "A combination of either '"+ + SOURCE_PARAM+ + "' + '"+ + DEST_PARAM+ + "', or '"+ + REPLACEMENT_PARAM+ + "' + '"+ + PATTERN_PARAM+ + "' init params are mandatory");+ }++ Object modelParam = args.remove(MODEL_PARAM);+ if (null == modelParam) {+ throw new SolrException(SERVER_ERROR, "Missing required init param '" + MODEL_PARAM + "'");+ }+ if (!(modelParam instanceof CharSequence)) {+ throw new SolrException(SERVER_ERROR, "Init param '" + MODEL_PARAM + "' must be a <str>");+ }+ model = modelParam.toString();+ log.info("In OpenNLP doccat - model: {}", model);++ Object vocabParam = args.remove(VOCAB_PARAM);+ if (null == vocabParam) {+ throw new SolrException(SERVER_ERROR, "Missing required init param '" + VOCAB_PARAM + "'");+ }+ if (!(vocabParam instanceof CharSequence)) {+ throw new SolrException(SERVER_ERROR, "Init param '" + VOCAB_PARAM + "' must be a <str>");+ }+ vocab = vocabParam.toString();++ if (0 < args.size()) {+ throw new SolrException(SERVER_ERROR, "Unexpected init param(s): '" + args.getName(0) + "'");+ }++ super.init(args);+ }++ /**+ * init helper method that should only be called when we know for certain that both the "source"+ * and "dest" init params do <em>not</em> exist.+ */+ private void initSimpleRegexReplacement(NamedList<?> args) {+ // The syntactic sugar for the case where there is only one regex pattern for source and the+ // same pattern+ // is used for the destination pattern...+ //+ // pattern != null && replacement != null+ //+ // ...as top level elements, with no other config options specified++ // if we got here we know we had pattern and replacement, now check for the other two so that+ // we can give a better+ // message than "unexpected"+ if (0 <= args.indexOf(SOURCE_PARAM, 0) || 0 <= args.indexOf(DEST_PARAM, 0)) {+ throw new SolrException(+ SERVER_ERROR,+ "Short hand syntax must not be mixed with full syntax. Found "+ + PATTERN_PARAM+ + " and "+ + REPLACEMENT_PARAM+ + " but also found "+ + SOURCE_PARAM+ + " or "+ + DEST_PARAM);+ }++ assert args.indexOf(SOURCE_PARAM, 0) < 0;++ Object patt = args.remove(PATTERN_PARAM);+ Object replacement = args.remove(REPLACEMENT_PARAM);++ if (null == patt || null == replacement) {+ throw new SolrException(+ SERVER_ERROR,+ "Init params '"+ + PATTERN_PARAM+ + "' and '"+ + REPLACEMENT_PARAM+ + "' are both mandatory if '"+ + SOURCE_PARAM+ + "' and '"+ + DEST_PARAM+ + "' are not both specified");+ }++ if (0 != args.size()) {+ throw new SolrException(+ SERVER_ERROR,+ "Init params '"+ + REPLACEMENT_PARAM+ + "' and '"+ + PATTERN_PARAM+ + "' must be children of '"+ + DEST_PARAM+ + "' to be combined with other options.");+ }++ if (!(replacement instanceof String)) {+ throw new SolrException(+ SERVER_ERROR, "Init param '" + REPLACEMENT_PARAM + "' must be a string (i.e. <str>)");+ }+ if (!(patt instanceof String)) {+ throw new SolrException(+ SERVER_ERROR, "Init param '" + PATTERN_PARAM + "' must be a string (i.e. <str>)");+ }++ dest = replacement.toString();+ try {+ this.pattern = Pattern.compile(patt.toString());+ } catch (PatternSyntaxException pe) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param " + PATTERN_PARAM + " is not a valid regex pattern: " + patt,+ pe);+ }+ srcInclusions = new SelectorParams();+ srcInclusions.fieldRegex = Collections.singletonList(this.pattern);+ }++ /**+ * init helper method that should only be called when we know for certain that both the "source"+ * and "dest" init params <em>do</em> exist.+ */+ private void initSourceSelectorSyntax(NamedList<?> args) {+ // Full and complete syntax where source and dest are mandatory.+ //+ // source may be a single string or a selector.+ // dest may be a single string or list containing pattern and replacement+ //+ // source != null && dest != null++ // if we got here we know we had source and dest, now check for the other two so that we can+ // give a better+ // message than "unexpected"+ if (0 <= args.indexOf(PATTERN_PARAM, 0) || 0 <= args.indexOf(REPLACEMENT_PARAM, 0)) {+ throw new SolrException(+ SERVER_ERROR,+ "Short hand syntax must not be mixed with full syntax. Found "+ + SOURCE_PARAM+ + " and "+ + DEST_PARAM+ + " but also found "+ + PATTERN_PARAM+ + " or "+ + REPLACEMENT_PARAM);+ }++ Object d = args.remove(DEST_PARAM);+ assert null != d;++ List<?> sources = args.getAll(SOURCE_PARAM);+ assert null != sources;++ if (1 == sources.size()) {+ if (sources.get(0) instanceof NamedList) {+ // nested set of selector options+ NamedList<?> selectorConfig = (NamedList<?>) args.remove(SOURCE_PARAM);++ srcInclusions = parseSelectorParams(selectorConfig);++ List<?> excList = selectorConfig.getAll("exclude");++ for (Object excObj : excList) {+ if (null == excObj) {+ throw new SolrException(+ SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' can not be null");+ }+ if (!(excObj instanceof NamedList)) {+ throw new SolrException(+ SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' must be <lst/>");+ }+ NamedList<?> exc = (NamedList<?>) excObj;+ srcExclusions.add(parseSelectorParams(exc));+ if (0 < exc.size()) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + SOURCE_PARAM+ + "' has unexpected 'exclude' sub-param(s): '"+ + selectorConfig.getName(0)+ + "'");+ }+ // call once per instance+ selectorConfig.remove("exclude");+ }++ if (0 < selectorConfig.size()) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + SOURCE_PARAM+ + "' contains unexpected child param(s): '"+ + selectorConfig.getName(0)+ + "'");+ }+ // consume from the named list so it doesn't interfere with subsequent processing+ sources.remove(0);+ }+ }+ if (1 <= sources.size()) {+ // source better be one or more strings+ srcInclusions.fieldName = new HashSet<>(args.removeConfigArgs("source"));+ }+ if (srcInclusions == null) {+ throw new SolrException(+ SERVER_ERROR,+ "Init params do not specify any field from which to extract entities, please supply either "+ + SOURCE_PARAM+ + " and "+ + DEST_PARAM+ + " or "+ + PATTERN_PARAM+ + " and "+ + REPLACEMENT_PARAM+ + ". See javadocs"+ + "for OpenNLPExtractNamedEntitiesUpdateProcessor for further details.");+ }++ if (d instanceof NamedList) {+ NamedList<?> destList = (NamedList<?>) d;++ Object patt = destList.remove(PATTERN_PARAM);+ Object replacement = destList.remove(REPLACEMENT_PARAM);++ if (null == patt || null == replacement) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + DEST_PARAM+ + "' children '"+ + PATTERN_PARAM+ + "' and '"+ + REPLACEMENT_PARAM+ + "' are both mandatory and can not be null");+ }+ if (!(patt instanceof String && replacement instanceof String)) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + DEST_PARAM+ + "' children '"+ + PATTERN_PARAM+ + "' and '"+ + REPLACEMENT_PARAM+ + "' must both be strings (i.e. <str>)");+ }+ if (0 != destList.size()) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + DEST_PARAM+ + "' has unexpected children: '"+ + destList.getName(0)+ + "'");+ }++ try {+ this.pattern = Pattern.compile(patt.toString());+ } catch (PatternSyntaxException pe) {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + DEST_PARAM+ + "' child '"+ + PATTERN_PARAM+ + " is not a valid regex pattern: "+ + patt,+ pe);+ }+ dest = replacement.toString();++ } else if (d instanceof String) {+ dest = d.toString();+ } else {+ throw new SolrException(+ SERVER_ERROR,+ "Init param '"+ + DEST_PARAM+ + "' must either be a string "+ + "(i.e. <str>) or a list (i.e. <lst>) containing '"+ + PATTERN_PARAM+ + "' and '"+ + REPLACEMENT_PARAM);+ }+ }++ @Override+ public void inform(final SolrCore core) {+ this.solrHome = core.getCoreContainer().getSolrHome();+ srcSelector =+ FieldMutatingUpdateProcessor.createFieldNameSelector(+ core.getResourceLoader(),+ core,+ srcInclusions,+ FieldMutatingUpdateProcessor.SELECT_NO_FIELDS);++ for (SelectorParams exc : srcExclusions) {+ srcSelector =+ FieldMutatingUpdateProcessor.wrap(+ srcSelector,+ FieldMutatingUpdateProcessor.createFieldNameSelector(+ core.getResourceLoader(),+ core,+ exc,+ FieldMutatingUpdateProcessor.SELECT_NO_FIELDS));+ }+ }++ @Override+ public final UpdateRequestProcessor getInstance(+ SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {+ final FieldNameSelector srcSelector = getSourceSelector();+ return new UpdateRequestProcessor(next) {++ DocumentCategorizerDL documentCategorizerDL = null;++ {+ // Initialize the categorizer.+ FileStore fs = req.getCoreContainer().getFileStore();++ var path = solrHome.resolve(ClusterFileStore.FILESTORE_DIRECTORY);+ Path modelFile = Path.of(model);+ Path vocabFile = Path.of(vocab);++ if (!Files.exists(modelFile)) {+ modelFile = path.resolve(model);+ }+ if (!Files.exists(vocabFile)) {+ vocabFile = path.resolve(vocab);+ }++ log.info("initializing the documentCategorizerDL");+ try {+ documentCategorizerDL =+ new DocumentCategorizerDL(+ modelFile.toFile(),+ vocabFile.toFile(),+ getCategories(),+ new AverageClassificationScoringStrategy(),+ new InferenceOptions());+ } catch (IOException e) {+ log.warn("Attempted to initialize documentCategorizerDL", e);+ } catch (OrtException e) {+ log.warn("Attempted to initialize documentCategorizerDL", e);+ }+ }++ @Override+ public void processAdd(AddUpdateCommand cmd) throws IOException {++ final SolrInputDocument doc = cmd.getSolrInputDocument();++ // Destination may be regex replace string, or "{EntityType}" replaced by+ // each entity's type, both of which can cause multiple output fields.+ Map<String, SolrInputField> destMap = new HashMap<>();++ // preserve initial values+ for (final String fname : doc.getFieldNames()) {+ if (!srcSelector.shouldMutate(fname)) continue;++ Collection<Object> srcFieldValues = doc.getFieldValues(fname);+ if (srcFieldValues == null || srcFieldValues.isEmpty()) continue;++ String resolvedDest = dest;++ if (pattern != null) {+ Matcher matcher = pattern.matcher(fname);+ if (matcher.find()) {+ resolvedDest = matcher.replaceAll(dest);+ } else {+ log.debug(+ "srcSelector.shouldMutate('{}') returned true, "+ + "but replacement pattern did not match, field skipped.",+ fname);+ continue;+ }+ }++ for (Object val : srcFieldValues) {+ for (Pair<String, String> entity : classify(val)) {+ SolrInputField destField = null;+ // String classification = entity.first();+ String classificationValue = entity.second();+ final String resolved = resolvedDest;+ if (doc.containsKey(resolved)) {+ destField = doc.getField(resolved);+ } else {+ SolrInputField targetField = destMap.get(resolved);+ if (targetField == null) {+ destField = new SolrInputField(resolved);+ } else {+ destField = targetField;+ }+ }+ destField.addValue(classificationValue);++ // put it in map to avoid concurrent modification...+ destMap.put(resolved, destField);+ }+ }+ }++ for (Map.Entry<String, SolrInputField> entry : destMap.entrySet()) {+ doc.put(entry.getKey(), entry.getValue());+ }+ super.processAdd(cmd);+ }++ private List<Pair<String, String>> classify(Object srcFieldValue) {++ String fullText = srcFieldValue.toString();++ // Send the fullText to the model for classification.+ log.info("calling categorizer()");+ final double[] result = documentCategorizerDL.categorize(new String[] {fullText});++ // Add the categories to the list and return it.+ // Just take the top category for now.+ // TODO: Allow for a threshold value for returning categories.++ List<Pair<String, String>> classifications = new ArrayList<>();++ String bestCategory = documentCategorizerDL.getBestCategory(result);+ if (log.isInfoEnabled()) {+ log.info("best category = {}", bestCategory);+ }++ Pair<String, String> pair = new Pair<>("classification", bestCategory);+ classifications.add(pair);++ return classifications;+ }+ };+ }++ /** macro */+ private static SelectorParams parseSelectorParams(NamedList<?> args) {+ return FieldMutatingUpdateProcessorFactory.parseSelectorParams(args);+ }++ private Map<Integer, String> getCategories() {++ // TODO: Read these from the Solr config for the processor.++ final Map<Integer, String> categories = new HashMap<>();++ categories.put(0, "very bad");+ categories.put(1, "bad");+ categories.put(2, "neutral");+ categories.put(3, "good");+ categories.put(4, "very good");++ return categories;+ }+}diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.bin b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.binnew file mode 100644index 00000000000..1e489fb4ffe--- /dev/null+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-model.bin@@ -0,0 +1,19 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++MOCK_MODEL_FILE_FOR_DOCUMENT_CATEGORIZER_TESTS+This is a placeholder file and not an actual model file.+In a real test environment, this would be replaced with a proper model file+or the tests would be modified to mock the file loading operations.diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txt b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txtnew file mode 100644index 00000000000..508ced5ee90--- /dev/null+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/doccat-vocab.txt@@ -0,0 +1,34 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Mock vocabulary file for DocumentCategorizer tests+good+bad+excellent+terrible+fine+average+neutral+very+love+product+recommend+quality+service+broke+immediately+special+performance+acceptablediff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xmlnew file mode 100644index 00000000000..4aa89d15528--- /dev/null+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/schema-document-categorizer.xml@@ -0,0 +1,57 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<schema name="document-categorizer" version="1.6">+ <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true"/>+ <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true"/>+ <fieldType name="text" class="solr.TextField" positionIncrementGap="100">+ <analyzer>+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ </analyzer>+ </fieldType>++ <!-- For document categorization -->+ <fieldType name="categorizer_text" class="solr.TextField" positionIncrementGap="100">+ <analyzer>+ <tokenizer class="solr.StandardTokenizerFactory"/>+ <filter class="solr.LowerCaseFilterFactory"/>+ <!--filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/-->+ </analyzer>+ </fieldType>++ <!-- Fields used in tests -->+ <field name="id" type="string" indexed="true" stored="true" required="true"/>+ <field name="text_s" type="string" indexed="true" stored="true"/>+ <field name="text1_s" type="string" indexed="true" stored="true"/>+ <field name="text2_s" type="string" indexed="true" stored="true"/>+ <field name="text0_s" type="string" indexed="true" stored="true"/>+ <field name="ignore_s" type="string" indexed="true" stored="true"/>+ <field name="review_text_s" type="string" indexed="true" stored="true"/>+ <field name="comments_s" type="string" indexed="true" stored="true"/>+ <field name="review_1_text" type="string" indexed="true" stored="true"/>+ <field name="review_2_text" type="string" indexed="true" stored="true"/>++ <!-- Destination fields -->+ <field name="category_s" type="strings" indexed="true" stored="true" multiValued="true"/>+ <field name="result_1_category" type="string" indexed="true" stored="true"/>+ <field name="result_2_category" type="string" indexed="true" stored="true"/>++ <uniqueKey>id</uniqueKey>+</schema>diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xmlnew file mode 100644index 00000000000..c40f3e17f3c--- /dev/null+++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-document-categorizer.xml@@ -0,0 +1,85 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<config>+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>+ <requestHandler name="/select" class="solr.SearchHandler"></requestHandler>+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>+ <schemaFactory class="ClassicIndexSchemaFactory"/>++ <updateRequestProcessorChain name="categorize-single">+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">+ <str name="modelFile">doccat-model.bin</str>+ <str name="vocabFile">doccat-vocab.txt</str>+ <str name="source">text_s</str>+ <str name="dest">category_s</str>+ </processor>+ </updateRequestProcessorChain>++ <updateRequestProcessorChain name="categorize-multi">+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">+ <str name="modelFile">doccat-model.bin</str>+ <str name="vocabFile">doccat-vocab.txt</str>+ <str name="source">text1_s</str>+ <str name="source">text2_s</str>+ <str name="dest">category_s</str>+ </processor>+ </updateRequestProcessorChain>++ <updateRequestProcessorChain name="categorize-regex">+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">+ <str name="modelFile">doccat-model.bin</str>+ <str name="vocabFile">doccat-vocab.txt</str>+ <lst name="source">+ <str name="fieldRegex">.*_text_s|comments_s</str>+ </lst>+ <str name="dest">category_s</str>+ </processor>+ </updateRequestProcessorChain>++ <updateRequestProcessorChain name="categorize-field-regex">+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">+ <str name="modelFile">doccat-model.bin</str>+ <str name="vocabFile">doccat-vocab.txt</str>+ <lst name="source">+ <str name="fieldRegex">review_\d_text</str>+ </lst>+ <lst name="dest">+ <str name="pattern">review_(\d)_text</str>+ <str name="replacement">result_$1_category</str>+ </lst>+ </processor>+ </updateRequestProcessorChain>++ <updateRequestProcessorChain name="categorize-selector">+ <processor class="solr.DocumentCategorizerUpdateProcessorFactory">+ <str name="modelFile">doccat-model.bin</str>+ <str name="vocabFile">doccat-vocab.txt</str>+ <lst name="source">+ <str name="fieldRegex">text\d_s</str>+ <lst name="exclude">+ <str name="fieldRegex">text0_s</str>+ </lst>+ </lst>+ <str name="dest">category_s</str>+ </processor>+ </updateRequestProcessorChain>+</config>\ No newline at end of filediff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.javanew file mode 100644index 00000000000..74af837305c--- /dev/null+++ b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestConfiguringDocumentCategorizerUpdateProcessorFactory.java@@ -0,0 +1,172 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.update.processor;++import java.nio.file.Path;+import org.apache.commons.io.file.PathUtils;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.NamedList;+import org.junit.BeforeClass;+import org.junit.Test;++/**+ * Tests for {@link DocumentCategorizerUpdateProcessorFactory}+ *+ * <p>Focused on verifying how DocumentCategorizerUpdateProcessorFactory is set up to run.+ *+ * <p>See test_opennlp.bats for a complete system test of DocumentCategorizerUpdateProcessorFactory.+ */+public class TestConfiguringDocumentCategorizerUpdateProcessorFactory+ extends UpdateProcessorTestBase {++ @BeforeClass+ public static void beforeClass() throws Exception {+ Path testHome = createTempDir();+ PathUtils.copyDirectory(getFile("analysis-extras/solr"), testHome);+ initCore("solrconfig-document-categorizer.xml", "schema-document-categorizer.xml", testHome);+ }++ @Test+ public void testBasicConfiguration() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Add source and dest+ args.add("source", "text_field");+ args.add("dest", "category_field");++ // Add required model files+ args.add("modelFile", "doccat-model.bin");+ args.add("vocabFile", "doccat-vocab.txt");++ // Should initialize without errors+ factory.init(args);+ }++ @Test(expected = SolrException.class)+ public void testMissingModelFile() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Add source and dest but miss modelFile+ args.add("source", "text_field");+ args.add("dest", "category_field");+ args.add("vocabFile", "doccat-vocab.txt");++ factory.init(args);+ }++ @Test(expected = SolrException.class)+ public void testMissingVocabFile() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Add source and dest but miss vocabFile+ args.add("source", "text_field");+ args.add("dest", "category_field");+ args.add("modelFile", "doccat-model.bin");++ factory.init(args);+ }++ @Test(expected = SolrException.class)+ public void testMissingSourceAndDest() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Add only model files but missing source/dest+ args.add("modelFile", "doccat-model.bin");+ args.add("vocabFile", "doccat-vocab.txt");++ factory.init(args);+ }++ @Test+ public void testRegexConfiguration() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Add source parameter+ NamedList<Object> sourceArgs = new NamedList<>();+ sourceArgs.add("fieldRegex", "text_.*");+ args.add("source", sourceArgs);++ // Create pattern and replacement as part of a dest structure+ NamedList<Object> destArgs = new NamedList<>();+ destArgs.add("pattern", "text_(.*)");+ destArgs.add("replacement", "category_$1");+ args.add("dest", destArgs);++ // Add required model files+ args.add("modelFile", "doccat-model.bin");+ args.add("vocabFile", "doccat-vocab.txt");++ // Should initialize without errors+ factory.init(args);+ }++ @Test+ public void testSourceSelector() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Create a complex source selector+ NamedList<Object> sourceArgs = new NamedList<>();+ sourceArgs.add("fieldRegex", "text_.*");+ NamedList<Object> excludeArgs = new NamedList<>();+ excludeArgs.add("fieldRegex", "text_ignore_.*");+ sourceArgs.add("exclude", excludeArgs);++ args.add("source", sourceArgs);+ args.add("dest", "category_field");+ args.add("modelFile", "doccat-model.bin");+ args.add("vocabFile", "doccat-vocab.txt");++ // Should initialize without errors+ factory.init(args);+ }++ @Test+ public void testDestWithPatternReplacement() {+ DocumentCategorizerUpdateProcessorFactory factory =+ new DocumentCategorizerUpdateProcessorFactory();+ NamedList<Object> args = new NamedList<>();++ // Simple source+ args.add("source", "text_field");++ // Dest with pattern and replacement+ NamedList<Object> destArgs = new NamedList<>();+ destArgs.add("pattern", "text_(.*)");+ destArgs.add("replacement", "category_$1");+ args.add("dest", destArgs);++ // Add required model files+ args.add("modelFile", "doccat-model.bin");+ args.add("vocabFile", "doccat-vocab.txt");++ // Should initialize without errors+ factory.init(args);+ }+}diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfileindex a42495189a3..0aec085dd80 100644--- a/solr/modules/extraction/gradle.lockfile+++ b/solr/modules/extraction/gradle.lockfile@@ -146,7 +146,7 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ruorg.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.pdfbox:fontbox:2.0.26=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.pdfbox:jbig2-imageio:3.0.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.pdfbox:jempbox:1.8.16=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathdiff --git a/solr/modules/langid/gradle.lockfile b/solr/modules/langid/gradle.lockfileindex 09899293f8a..7b56fcac95b 100644--- a/solr/modules/langid/gradle.lockfile+++ b/solr/modules/langid/gradle.lockfile@@ -108,7 +108,7 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ruorg.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath-org.apache.opennlp:opennlp-tools:2.5.3=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath+org.apache.opennlp:opennlp-tools:2.5.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.tika:tika-core:1.28.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper-jute:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathdiff --git a/solr/modules/sql/gradle.lockfile b/solr/modules/sql/gradle.lockfileindex 89e85fd3495..713148901f9 100644--- a/solr/modules/sql/gradle.lockfile+++ b/solr/modules/sql/gradle.lockfile@@ -32,6 +32,7 @@ com.ibm.icu:icu4j:77.1=jarValidation,testRuntimeClasspathcom.j256.simplemagic:simplemagic:1.17=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcom.jayway.jsonpath:json-path:2.9.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcom.lmax:disruptor:3.4.4=solrPlatformLibs+com.microsoft.onnxruntime:onnxruntime:1.20.0=jarValidation,testRuntimeClasspathcom.tdunning:t-digest:3.3=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcommons-cli:commons-cli:1.10.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathcommons-codec:commons-codec:1.19.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath@@ -121,7 +122,8 @@ org.apache.lucene:lucene-spatial-extras:10.2.1=jarValidation,runtimeClasspath,ruorg.apache.lucene:lucene-spatial3d:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-suggest:10.2.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspathorg.apache.lucene:lucene-test-framework:10.2.1=jarValidation,testCompileClasspath,testRuntimeClasspath-org.apache.opennlp:opennlp-tools:2.5.3=jarValidation,testRuntimeClasspath+org.apache.opennlp:opennlp-dl:2.5.4=jarValidation,testRuntimeClasspath+org.apache.opennlp:opennlp-tools:2.5.4=jarValidation,testRuntimeClasspathorg.apache.zookeeper:zookeeper-jute:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathorg.apache.zookeeper:zookeeper:3.9.4=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspathorg.apiguardian:apiguardian-api:1.1.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspathdiff --git a/solr/packaging/test/test_opennlp.bats b/solr/packaging/test/test_opennlp.batsnew file mode 100644index 00000000000..c2fd6755b38--- /dev/null+++ b/solr/packaging/test/test_opennlp.bats@@ -0,0 +1,119 @@+#!/usr/bin/env bats++# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++load bats_helper++setup_file() {+ common_clean_setup++}++teardown_file() {+ common_setup+ solr stop --all+}++setup() {+ common_setup+}++teardown() {+ # save a snapshot of SOLR_HOME for failed tests+ save_home_on_failure+}++# This BATS style test is really just to help explore the space of Modern NLP in+# Apache Solr, versus a "true" integration test that I want to have run regularly.+# On the other hand, since integrating NLP requires a lot of steps, maybe having this+# long test as an "integration" test is something we decide is okay?+# I also have dreams of incorporating this as code snippets in a Tutorial via the ascii doc tags+# like we use for the SolrJ code snippets. That way we know the snippets continue to work!+@test "Check lifecycle of sentiment classification" {++ echo "Downloading onnx model and vocab..."++ mkdir -p ${SOLR_TIP}/models/sentiment/++ # Models originally sourced from Huggingface and store on ASF hardware+ wget -O ${SOLR_TIP}/models/sentiment/model.onnx https://nightlies.apache.org/solr/opennlp-dl-integration/model_quantized.onnx+ wget -O ${SOLR_TIP}/models/sentiment/vocab.txt https://nightlies.apache.org/solr/opennlp-dl-integration/vocab.txt+ # wget -O ${SOLR_TIP}/models/sentiment/model.onnx https://huggingface.co/onnx-community/bert-base-multilingual-uncased-sentiment-ONNX/resolve/main/onnx/model_quantized.onnx+ # wget -O ${SOLR_TIP}/models/sentiment/vocab.txt https://huggingface.co/onnx-community/bert-base-multilingual-uncased-sentiment-ONNX/raw/main/vocab.txt++ run ls -alh ${SOLR_TIP}/modules/analysis-extras/lib+ refute_output --partial "onnxruntime_gpu"++ # Can't figure out magic policy stuff to allow loading ONNX, so disable security manager.+ export SOLR_SECURITY_MANAGER_ENABLED=false++ solr start -m 4g -Dsolr.modules=analysis-extras -Denable.packages=true+ solr assert --started http://localhost:${SOLR_PORT}/solr --timeout 5000++ run solr create -c COLL_NAME+ assert_output --partial "Created collection 'COLL_NAME'"++ curl -X POST -H 'Content-type:application/json' --data-binary '{+ "add-field":{+ "name":"name",+ "type":"string",+ "stored":true }+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/schema"++ curl -X POST -H 'Content-type:application/json' --data-binary '{+ "add-field":{+ "name":"name_sentiment",+ "type":"string",+ "stored":true }+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/schema"++ run curl --data-binary @${SOLR_TIP}/models/sentiment/vocab.txt -X PUT "http://localhost:${SOLR_PORT}/api/cluster/filestore/files/models/sentiment/vocab.txt"+ assert_output --partial '"status":0'++ run curl --data-binary @${SOLR_TIP}/models/sentiment/model.onnx -X PUT "http://localhost:${SOLR_PORT}/api/cluster/filestore/files/models/sentiment/model.onnx"+ assert_output --partial '"status":0'++ run curl -X POST -H 'Content-type:application/json' -d '{+ "add-updateprocessor": {+ "name": "sentimentClassifier",+ "class": "solr.processor.DocumentCategorizerUpdateProcessorFactory",+ "modelFile": "models/sentiment/model.onnx",+ "vocabFile": "models/sentiment/vocab.txt",+ "source": "name",+ "dest": "name_sentiment"+ }+ }' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/config"+ assert_output --partial '"status":0'++ run curl -X POST -H 'Content-type:application/json' -d '[+ {+ "id":"good",+ "name" : "that was an awesome movie!"+ },+ {+ "id":"bad",+ "name" : "that movie was bad and terrible"+ }+ ]' "http://localhost:${SOLR_PORT}/solr/COLL_NAME/update/json?processor=sentimentClassifier&commit=true"++ assert_output --partial '"status":0'++ run curl -X GET "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=id:good"+ assert_output --partial '"name_sentiment":"very good"'++ run curl -X GET "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=id:bad"+ assert_output --partial '"name_sentiment":"very bad"'+}diff --git a/solr/solr-ref-guide/antora.yml b/solr/solr-ref-guide/antora.ymlindex 2882d5f4655..d612229ed36 100644--- a/solr/solr-ref-guide/antora.yml+++ b/solr/solr-ref-guide/antora.yml@@ -48,7 +48,7 @@ asciidoc:dep-version-commons-codec: '1.15'dep-version-dropwizard: '4.2.12'dep-version-log4j: '2.19.0'- dep-version-opennlp: '1.9.4'+ dep-version-opennlp: '2.5.4'dep-version-tika: '1.28.4'dep-version-zookeeper: '3.8.0'dep-version-lucene: '9.4.2'