Apache Solr open-source search software
SOLR-15455: Facilitate joint Solr/Lucene development via local dependency substitution (#304)
Dawid Weiss6046f561c92a
diff --git a/.gitignore b/.gitignoreindex 03cd8215bae..0ae47642375 100644--- a/.gitignore+++ b/.gitignore@@ -32,3 +32,6 @@ __pycache__# Emacs backup*~++# Ignore lucene included build+lucene/\ No newline at end of filediff --git a/build.gradle b/build.gradleindex dafeae7e2f0..88d730c845e 100644--- a/build.gradle+++ b/build.gradle@@ -164,6 +164,9 @@ apply from: file('gradle/testing/profiling.gradle')apply from: file('gradle/testing/beasting.gradle')apply from: file('gradle/help.gradle')+// Configures development for joint Lucene/ Solr composite build.+apply from: file('gradle/lucene-dev/lucene-dev-repo-composite.gradle')+// Ant-compatibility layer. ALL OF THESE SHOULD BE GONE at some point. They are// here so that we can coexist with current ant build but they are indicative// of potential problems with the build conventions, dependencies, etc.@@ -190,3 +193,5 @@ apply from: file('gradle/solr/packaging.gradle')apply from: file('gradle/solr/solr-forbidden-apis.gradle')apply from: file('gradle/ant-compat/solr.post-jar.gradle')++diff --git a/gradle/lucene-dev/lucene-dev-repo-composite.gradle b/gradle/lucene-dev/lucene-dev-repo-composite.gradlenew file mode 100644index 00000000000..0ad6ee8d79e--- /dev/null+++ b/gradle/lucene-dev/lucene-dev-repo-composite.gradle@@ -0,0 +1,142 @@+import java.util.function.BiFunction++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Local Lucene development repository resolution:+// 1) A "-Plucene.dev.version=[version]" property, resolving Lucene artifacts from a local Maven repository.+// 2) A non-empty property "-Plucene.dev.path=[path]" pointing to a local path. Relative paths+// are resolved against the root project directory.+// 3) An auto-wired 'lucene' subfolder, if present. To skip auto-wiring, pass+// a blank value in step 2: "-Plucene.dev.path=".+++// This script is applied in settings.gradle and later at build time: these two contexts+// are distinctively different and have separate (and limited) APIs.+def configuringSettings = (rootProject instanceof org.gradle.api.initialization.ProjectDescriptor)++// Accessor for -P properties from settings or at build time.+BiFunction<String, String, String> resolvePropertyValue = { propertyName, defValue ->+ if (configuringSettings) {+ return settings.startParameter.projectProperties.getOrDefault(propertyName, defValue)+ } else {+ return project.properties.getOrDefault(propertyName, defValue)+ }+}++def PROP_FORCE_VERSION="lucene.dev.version"+def PROP_FORCE_PATH="lucene.dev.path"+def READ_ACCESS_PROPERTY="lucene-dev-path.dir"+++def forcedLuceneVersion = resolvePropertyValue(PROP_FORCE_VERSION, null)+if (forcedLuceneVersion != null) {+ if (!configuringSettings) {+ logger.lifecycle("Lucene version forced by -P${PROP_FORCE_VERSION}=${forcedLuceneVersion}")++ allprojects {+ repositories {+ mavenLocal()+ }++ tasks.withType(Test) {+ def userHome = System.properties.get('user.home')+ systemProperty READ_ACCESS_PROPERTY, file("${userHome}/.m2/repository/org/apache/lucene").absolutePath+ }++ configurations.all {+ resolutionStrategy.eachDependency {+ if (requested.group == "org.apache.lucene") {+ useVersion(forcedLuceneVersion)+ because("Lucene version forced manually by 'lucene.dev.version' property.")+ }+ }+ }+ }+ }++ // Step 1: end resolution+ return+}++def luceneDevRepo = null+def defaultLuceneDevRepo = file("${rootDir}/lucene")+String propertyValue = resolvePropertyValue(PROP_FORCE_PATH, null)+if (propertyValue != null) {+ // Step 2.+ if (propertyValue.isBlank()) {+ if (defaultLuceneDevRepo.exists() && configuringSettings) {+ logger.lifecycle("Local Lucene development repository has been detected but won't be used.")+ }+ } else {+ // For relative path `propertyValue` spec, `file(propertyValue)` at settings configuration time+ // resolves relative to "${rootDir}/gradle/" (as opposed to "${rootDir}/"); so we use the Java+ // Path API here for consistent relative path resolution at configuration and build time+ luceneDevRepo = file(rootDir).toPath().resolve(propertyValue).normalize().toFile()+ if (!luceneDevRepo.exists()) {+ throw new GradleException("Lucene repository does not exist at: -P${PROP_FORCE_PATH}=${luceneDevRepo}.")+ }+ }+} else if (defaultLuceneDevRepo.exists()) {+ // Step 3+ luceneDevRepo = defaultLuceneDevRepo+}+++if (luceneDevRepo != null) {+ // Allow turning off this auto-wiring via -Dlucene.dev.repo=false (can't be a -P property because+ // at settings evaluation time we don't have project properties yet).+ if (configuringSettings) {+ // We substitute the exact version of Lucene we currently have in versions.props across all the dependencies.+ // We can't just substitute all references without looking at the versoin because+ // plugin dependencies then also get substituted and everything crashes.+ String luceneVersion = (file("${rootDir}/versions.props").getText("UTF-8") =~ /org.apache.lucene:\*=(.+)/)[0][1]+ logger.lifecycle("Local Lucene development repository will be used substituting ${luceneVersion}: ${luceneDevRepo}")++ // Include Lucene repository as a composite and substitute module names.+ includeBuild(luceneDevRepo) {+ dependencySubstitution {+ all { DependencySubstitution dependency ->+ if (dependency.requested instanceof ModuleComponentSelector) {+ def moduleSelector = (ModuleComponentSelector) dependency.requested+ if (moduleSelector.group == "org.apache.lucene" && moduleSelector.version == luceneVersion) {+ // Map Maven artifact name to lucene module name.+ def projectModuleName =+ moduleSelector.moduleIdentifier.name+ .replaceFirst("^lucene-analysis-", ":lucene:analysis:")+ .replaceFirst("^lucene-", ":lucene:")++ logger.info("Substituting Lucene dependency ${moduleSelector} with an included build project: ${projectModuleName}")+ dependency.useTarget(project(projectModuleName))+ }+ }+ }+ }+ }+ } else {+ // We're being applied at build-time and Lucene development repository exists. Configure+ // certain aspects of the build so that things work with it.++ // Security policy requires read access to the repo path.+ allprojects {+ tasks.withType(Test) {+ systemProperty READ_ACCESS_PROPERTY, luceneDevRepo.absolutePath+ }+ }+ }+}+diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policyindex 297647aa2ae..7232474fa66 100644--- a/gradle/testing/randomization/policies/solr-tests.policy+++ b/gradle/testing/randomization/policies/solr-tests.policy@@ -232,4 +232,6 @@ grant {permission java.io.FilePermission "${gradle.worker.jar}", "read";// Allow reading from classpath JARs (resources).permission java.io.FilePermission "${gradle.user.home}${/}-", "read";+ // Allow read access to Lucene jars if "-Plucene.dev.version=" or "-Plucene.dev.path=" is used+ permission java.io.FilePermission "${lucene-dev-path.dir}${/}-", "read";};diff --git a/gradle/validation/validate-source-patterns.gradle b/gradle/validation/validate-source-patterns.gradleindex 284fd99751a..f9639d0e00c 100644--- a/gradle/validation/validate-source-patterns.gradle+++ b/gradle/validation/validate-source-patterns.gradle@@ -120,6 +120,7 @@ configure(rootProject) {// default excludes.exclude '**/build/**'+ exclude '**/.idea/**'exclude 'dev-tools/solr-missing-doclet/src/**/*.java' // <-- TODO: remove once we allow "var" on master// ourselves :-)diff --git a/help/dependencies.txt b/help/dependencies.txtindex f84a04dbc67..d67130f2b87 100644--- a/help/dependencies.txt+++ b/help/dependencies.txt@@ -108,6 +108,40 @@ If you want to upgrade Lucene to a newer build proceed like the following:- Run: gradlew --write-locks (as described before)+Lucene local dependency substitution+-------------------------------++Because of the fundamental nature of Solr's dependency on Lucene, Solr+facilitates dependency substitution to support developing against a local+Lucene version (e.g., for joint Solr and Lucene development).++A local Lucene dependency substitution is resolved by evaluating the+following conditions, in priority order:++ 1) A "-Plucene.dev.version=[version]" property, resolving Lucene artifacts+ from a local Maven repository.+ 2) A non-empty property "-Plucene.dev.path=[path]" pointing to a local+ path. Relative paths are resolved against the Solr root project+ directory.+ 3) An auto-wired 'lucene' subdirectory, if present. To skip auto-wiring,+ pass a blank value for condition 2: "-Plucene.dev.path=".++Note: Lucene dependencies substituted via conditions 2 and 3 require the+local Lucene repository to use the same Gradle version as the Solr root+project.++Local Maven repository Lucene artifacts (for use with the+"-Plucene.dev.version" project property) may be created by running the+following command from within a local Lucene project:++ gradlew mavenToLocalRepo++As is the case with project properties generally, the relevant properties+can be invoked on the command line, or configured via the gradle.properties+file (the latter approach is particularly useful for making IDEs aware of+associated dependency substitutions).++Inspecting current dependencies-------------------------------diff --git a/settings.gradle b/settings.gradleindex 1f68a4a5174..a04cd7c591b 100644--- a/settings.gradle+++ b/settings.gradle@@ -45,7 +45,9 @@ include "solr:benchmark"include "solr:test-framework"include "solr:solr-ref-guide"include "solr:example"-include "solr:documentation"include "solr:packaging"include "solr:docker"++// Configures development for joint Lucene/ Solr composite build.+apply from: file('gradle/lucene-dev/lucene-dev-repo-composite.gradle')diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 5dc9f7a368f..91f0eb68676 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -370,6 +370,8 @@ Other Changes* SOLR-15692: Fix parsing of -zkHost flag for the bin/solr commands (janhoy)+* SOLR-15455: Facilitate joint Solr/Lucene development via local dependency substitution (Dawid Weiss, Michael Gibney)+Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution