Apache Solr open-source search software
SOLR-16641 - Generate gradle.properties from gradlew (#1320)
Colvin Cowie50a064809902
diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.ymlindex 6daadd1817e..d8a6076da64 100644--- a/.github/workflows/bin-solr-test.yml+++ b/.github/workflows/bin-solr-test.yml@@ -34,8 +34,6 @@ jobs:restore-keys: |${{ runner.os }}-gradle-binsolr-${{ runner.os }}-gradle-- - name: Initialize gradle settings- run: ./gradlew localSettings- name: Test the bin/solr scriptrun: ./gradlew integrationTests- name: Archive logsdiff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.ymlindex 511cee11e8c..ddf6292d46f 100644--- a/.github/workflows/docker-test.yml+++ b/.github/workflows/docker-test.yml@@ -42,8 +42,6 @@ jobs:restore-keys: |${{ runner.os }}-gradle-docker-${{ runner.os }}-gradle-- - name: Initialize gradle settings- run: ./gradlew localSettings- name: Build Docker image with Gradlerun: ./gradlew solr:docker:docker- name: Run tests on Docker imagediff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.ymlindex 73bb5367605..6cc1425db84 100644--- a/.github/workflows/gradle-precommit.yml+++ b/.github/workflows/gradle-precommit.yml@@ -34,9 +34,6 @@ jobs:${{ runner.os }}-gradle-precommit-${{ runner.os }}-gradle-- - name: Initialize gradle settings- run: ./gradlew localSettings-- name: Run gradle check (without tests)run: ./gradlew check -x test -Ptask.times=truediff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.ymlindex a935a5598be..625ab7ee437 100644--- a/.github/workflows/solrj-test.yml+++ b/.github/workflows/solrj-test.yml@@ -33,7 +33,5 @@ jobs:restore-keys: |${{ runner.os }}-gradle-solrj-${{ runner.os }}-gradle-- - name: Initialize gradle settings- run: ./gradlew localSettings- name: Test the SolrJ Packagerun: ./gradlew solr:solrj:testdiff --git a/.gitignore b/.gitignoreindex 419ffff531c..18542bd60d4 100644--- a/.gitignore+++ b/.gitignore@@ -37,7 +37,7 @@ __pycache__*~# Ignore lucene included build-lucene/+/lucene# Ignore gradle wrappergradle/wrapper/gradle-wrapper.jardiff --git a/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java b/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.javanew file mode 100644index 00000000000..db4f804f12e--- /dev/null+++ b/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java@@ -0,0 +1,72 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.Paths;+import java.nio.file.StandardOpenOption;+import java.util.Map;++/**+ * Standalone class that generates a populated gradle.properties from a template.+ *+ * <p>Has no dependencies outside of standard java libraries+ */+public class GradlePropertiesGenerator {+ public static void main(String[] args) {+ if (args.length != 2) {+ System.err.println("Usage: java GradlePropertiesGenerator.java <source> <destination>");+ System.exit(2);+ }++ try {+ new GradlePropertiesGenerator().run(Paths.get(args[0]), Paths.get(args[1]));+ } catch (Exception e) {+ System.err.println("ERROR: " + e.getMessage());+ System.exit(3);+ }+ }++ public void run(Path source, Path destination) throws IOException {+ if (!Files.exists(source)) {+ throw new IOException("template file not found: " + source);+ }+ if (Files.exists(destination)) {+ System.out.println(destination + " already exists, skipping generation.");+ return;+ }++ // Approximate a common-sense default for running gradle/tests with parallel+ // workers: half the count of available cpus but not more than 12.+ var cpus = Runtime.getRuntime().availableProcessors();+ var maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12));+ var testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12));++ var replacements = Map.of("@MAX_WORKERS@", maxWorkers, "@TEST_JVMS@", testsJvms);++ System.out.println("Generating gradle.properties");+ String fileContent = Files.readString(source, StandardCharsets.UTF_8);+ for (var entry : replacements.entrySet()) {+ fileContent = fileContent.replace(entry.getKey(), String.valueOf(entry.getValue()));+ }+ Files.writeString(+ destination, fileContent, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW);+ }+}diff --git a/dev-docs/how-to-contribute.adoc b/dev-docs/how-to-contribute.adocindex ef82760399e..d072d69260c 100644--- a/dev-docs/how-to-contribute.adoc+++ b/dev-docs/how-to-contribute.adoc@@ -17,6 +17,7 @@ In order to make a new contribution to Solr you will use the fork you have creat1. Create a new Jira issue in the Solr project: https://issues.apache.org/jira/projects/SOLR/issues2. Create a new branch in your Solr fork to provide a PR for your contribution on the newly created issue. Make any necessary changes for the given bug/feature in that branch. You can use additional information in these dev-docs to build and test your code as well as ensure it passes code quality checks.3. Once you are satisfied with your changes, get your branch ready for a PR by running `./gradlew tidy updateLicenses check -x test`. This will format your source code, update licenses of any dependency version changes and run all pre-commit tests. Commit the changes.+* Note: the `check` command requires `perl` and `python3` to be present on your `PATH` to validate documentation.4. Open a PR of your branch against the `main` branch of the apache/solr repository. When you open a PR on your fork, this should be the default option.* The title of your PR should include the Solr Jira issue that you opened, i.e. `SOLR-12345: New feature`.* The PR description will automatically populate with a pre-set template that you will need to fill out.diff --git a/dev-docs/solr-source-code.adoc b/dev-docs/solr-source-code.adocindex 9f9cf79e021..9daf86cd2d4 100644--- a/dev-docs/solr-source-code.adoc+++ b/dev-docs/solr-source-code.adoc@@ -38,6 +38,8 @@ If you want to build the documentation, type `./gradlew -p solr documentation`.`./gradlew check` will assemble Solr and run all validation tasks unit tests.+NOTE: the `check` command requires `perl` and `python3` to be present on your `PATH` to validate documentation.+To build the final Solr artifacts run `./gradlew assemble`.Lastly, there is developer oriented documentation in `./dev-docs/README.adoc` thatdiff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yamlindex 747c62d22a4..7e6875889aa 100644--- a/dev-tools/scripts/releaseWizard.yaml+++ b/dev-tools/scripts/releaseWizard.yaml@@ -356,8 +356,6 @@ groups:- !Commandcmd: git pull --ff-onlystdout: true- - !Command- cmd: "{{ gradle_cmd }} localSettings"- !Commandcmd: "{{ gradle_cmd }} clean check -x test"- !Tododiff --git a/gradle/generation/local-settings.gradle b/gradle/generation/local-settings.gradleindex 263183b57e7..e164b890706 100644--- a/gradle/generation/local-settings.gradle+++ b/gradle/generation/local-settings.gradle@@ -41,87 +41,12 @@ configure(rootProject) {throw new GradleException("Certain gradle tasks and plugins require access to jdk.compiler" +" internals, your gradle.properties might have just been generated or could be" +- " out of sync (see help/localSettings.txt)")- }- }- }-- task localSettings() {- doFirst {- // If we don't have the defaults yet, create them.- if (hasDefaults) {- logger.lifecycle("Local settings already exist, skipping generation.")- } else {- // Approximate a common-sense default for running gradle/tests with parallel- // workers: half the count of available cpus but not more than 12.- def cpus = Runtime.runtime.availableProcessors()- def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))- def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))-- // Write the defaults for this machine.- rootProject.file("gradle.properties").write("""-# These settings have been generated automatically on the first run.-# See gradlew :helpLocalSettings for more information.-systemProp.file.encoding=UTF-8--# Set up gradle JVM defaults.-#-# We also open up internal compiler modules for spotless/ google java format.-org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \\- --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \\- --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \\- --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \\- --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \\- --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED--# Run at normal priority, in parallel-org.gradle.parallel=true-org.gradle.priority=normal--# This setting enables local task output caches. This will speed up-# your local builds in most cases but will also consume disk space in your-# gradle home. See SOLR-15603 for details.-# org.gradle.caching=true--# Silence gradle warnings. We'll deal with them when we upgrade the wrapper.-org.gradle.warning.mode=none--# You may disable the background daemon if it consumes too much memory.-org.gradle.daemon=true-# timeout after 15 mins of inactivity.-org.gradle.daemon.idletimeout=900000--# Maximum number of parallel gradle workers.-org.gradle.workers.max=${maxWorkers}--# Maximum number of test JVMs forked per test task.-tests.jvms=${testsJvms}--# Disable auto JVM provisioning (we don't use toolchains yet but want no surprises).-org.gradle.java.installations.auto-download=false--# Set these to enable automatic JVM location discovery.-org.gradle.java.installations.fromEnv=JDK11,JDK12,JDK13,JDK14,JDK15,JDK16,JDK17-org.gradle.java.installations.paths=(custom paths)--""", "UTF-8")-- logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " +- "I wrote some sane defaults (for this machine) to 'gradle.properties', " +- "they will be picked up on consecutive gradle invocations (not this one).\n\n" +- "Run gradlew :helpLocalSettings for more information.")+ " out of sync (see gradle/template.gradle.properties)")}}}}-if (!hasDefaults) {- // Make all tasks depend on local setup to make sure it'll run.- allprojects {- tasks.all { task ->- if (task != rootProject.localSettings) {- task.dependsOn rootProject.localSettings- }- }- }+task localSettings() {+ // This is just a placeholder until all references to the localSettings task are removed #SOLR-16641}diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.propertiesnew file mode 100644index 00000000000..c52e3048a77--- /dev/null+++ b/gradle/template.gradle.properties@@ -0,0 +1,102 @@+#############################+# Local developer settings #+#############################+#+# The first invocation of any task in Solr gradlew will generate and save this project-local 'gradle.properties' file.+# This file contains the defaults you may (but don't have to) tweak for your particular hardware (or taste). Note there+# are certain settings in that file that may be _required_ at runtime for certain plugins (an example is the spotless/+# google java format plugin, which requires adding custom exports to JVM modules). gradlew only generates this file+# if it's not already present (it never overwrites the defaults) -- occasionally you may have to manually delete (or move)+# this file and regenerate from scratch.+#+# This is an overview of some of these settings.+#+###############+# Parallelism #+###############+#+# Gradle build can run tasks in parallel but by default it consumes all CPU cores which+# is too optimistic a default for Solr tests. You can disable the parallelism+# entirely or assign it a 'low' priority with these properties:+#+# org.gradle.parallel=[true, false]+# org.gradle.priority=[normal, low]+#+# The default level of parallelism is computed based on the number of cores on+# your machine (on the first run of gradle build). By default these are fairly conservative+# settings (half the number of cores for workers, for example):+#+# org.gradle.workers.max=[X]+# tests.jvms=[N <= X]+#+# The number of test JVMs can be lower than the number of workers: this just means+# that two projects can run tests in parallel to saturate all the workers. The I/O and memory+# bandwidth limits will kick in quickly so even if you have a very beefy machine bumping+# it too high may not help.+#+# You can always override these settings locally using command line as well:+# gradlew -Ptests.jvms=N --max-workers=X+#+#############+# Test JVMS #+#############+#+# Test JVMs have their own set of arguments which can be customized. These are configured+# separately from the gradle workers, for example:+#+# tests.jvms=3+# tests.heapsize=512m+# tests.minheapsize=512m+# tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1+#+#################+# Gradle Daemon #+#################+#+# The gradle daemon is a background process that keeps an evaluated copy of the project+# structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't+# like the idea of having a (sizeable) background process running in the background,+# disable it.+#+# org.gradle.daemon=[true, false]+# org.gradle.jvmargs=...+#############################################################################################++# UTF-8 as standard file encoding+systemProp.file.encoding=UTF-8++# Set up gradle JVM defaults.+#+# We also open up internal compiler modules for spotless/ google java format.+org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \+ --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \+ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \+ --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \+ --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \+ --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED++# Run at normal priority, in parallel+org.gradle.parallel=true+org.gradle.priority=normal++# This setting enables local task output caches. This will speed up+# your local builds in most cases but will also consume disk space in your+# gradle home. See SOLR-15603 for details.+# org.gradle.caching=true++# Silence gradle warnings. We'll deal with them when we upgrade the wrapper.+org.gradle.warning.mode=none++# You may disable the background daemon if it consumes too much memory.+org.gradle.daemon=true+# timeout after 15 mins of inactivity.+org.gradle.daemon.idletimeout=900000++# Maximum number of parallel gradle workers.+org.gradle.workers.max=@MAX_WORKERS@++# Maximum number of test JVMs forked per test task.+tests.jvms=@TEST_JVMS@++# Disable auto JVM provisioning (we don't use toolchains yet but want no surprises).+org.gradle.java.installations.auto-download=falsediff --git a/gradle/validation/gradlew-scripts-tweaked.gradle b/gradle/validation/gradlew-scripts-tweaked.gradleindex a37d4e52b13..2720e70617e 100644--- a/gradle/validation/gradlew-scripts-tweaked.gradle+++ b/gradle/validation/gradlew-scripts-tweaked.gradle@@ -32,8 +32,8 @@ configure(rootProject) {doFirst {scripts.each { file ->def content = new String(file.readBytes(), StandardCharsets.US_ASCII)- if (content.indexOf("Don't fork a daemon mode on initial run that generates local defaults") < 0) {- throw new GradleException("Launch script ${file} does not have a manual daemon tweak (see LUCENE-9232).")+ if (content.indexOf("GradlePropertiesGenerator") < 0) {+ throw new GradleException("Launch script ${file} does not have a tweak to generate gradle.properties (see SOLR-16641).")}}}diff --git a/gradlew b/gradlewindex 59dd3f7eb88..03f6fc89ee0 100755--- a/gradlew+++ b/gradlew@@ -163,11 +163,16 @@ fiCLASSPATH=$GRADLE_WRAPPER_JAR-# Don't fork a daemon mode on initial run that generates local defaults.-GRADLE_DAEMON_CTRL=+# START OF SOLR CUSTOMIZATION+# Generate gradle.properties if they don't existif [ ! -e "$APP_HOME/gradle.properties" ]; then- GRADLE_DAEMON_CTRL=--no-daemon+ "$JAVACMD" $JAVA_OPTS --source 11 "$APP_HOME/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "$APP_HOME/gradle/template.gradle.properties" "$APP_HOME/gradle.properties"+ GENERATOR_STATUS=$?+ if [ "$GENERATOR_STATUS" -ne 0 ]; then+ exit $GENERATOR_STATUS+ fifi+# END OF SOLR CUSTOMIZATION# Increase the maximum file descriptors if we can.if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then@@ -237,7 +242,6 @@ set -- \"-Dorg.gradle.appname=$APP_BASE_NAME" \-classpath "$CLASSPATH" \org.gradle.wrapper.GradleWrapperMain \- $GRADLE_DAEMON_CTRL \"$@"# Stop when "xargs" is not available.diff --git a/gradlew.bat b/gradlew.batindex 964ab6aced2..dc413985442 100644--- a/gradlew.bat+++ b/gradlew.bat@@ -80,12 +80,19 @@ IF %ERRORLEVEL% NEQ 0 goto fail@rem Setup the command lineset CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar-@rem Don't fork a daemon mode on initial run that generates local defaults.-SET GRADLE_DAEMON_CTRL=-IF NOT EXIST "%DIRNAME%\gradle.properties" SET GRADLE_DAEMON_CTRL=--no-daemon+@rem START OF SOLR CUSTOMIZATION+@rem Generate gradle.properties if they don't exist+IF NOT EXIST "%APP_HOME%\gradle.properties" (+ @rem local expansion is needed to check ERRORLEVEL inside control blocks.+ setlocal enableDelayedExpansion+ "%JAVA_EXE%" --source 11 "%APP_HOME%/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"+ IF %ERRORLEVEL% NEQ 0 goto fail+ endlocal+)+@rem END OF SOLR CUSTOMIZATION@rem Execute Gradle-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %GRADLE_DAEMON_CTRL% %*+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*:end@rem End local scope for the variables with windows NT shelldiff --git a/help/localSettings.txt b/help/localSettings.txtindex b4510563a7c..5089bb669ca 100644--- a/help/localSettings.txt+++ b/help/localSettings.txt@@ -2,61 +2,7 @@ Local developer settings========================The first invocation of any task in Solr gradle build will generate-and save a project-local 'gradle.properties' file. This file contains-the defaults you may (but don't have to) tweak for your particular hardware-(or taste). Note there are certain settings in that file that may-be _required_ at runtime for certain plugins (an example is the spotless/-google java format plugin, which requires adding custom exports to JVM modules). Gradle-build only generates this file if it's not already present (it never overwrites-the defaults) -- occasionally you may have to manually delete (or move) this-file and regenerate from scratch.--This is an overview of some of these settings.--Parallelism--------------Gradle build can run tasks in parallel but by default it consumes all CPU cores which-is too optimistic a default for Solr tests. You can disable the parallelism-entirely or assign it a 'low' priority with these properties:--org.gradle.parallel=[true, false]-org.gradle.priority=[normal, low]--The default level of parallelism is computed based on the number of cores on-your machine (on the first run of gradle build). By default these are fairly conservative-settings (half the number of cores for workers, for example):--org.gradle.workers.max=[X]-tests.jvms=[N <= X]--The number of test JVMs can be lower than the number of workers: this just means-that two projects can run tests in parallel to saturate all the workers. The I/O and memory-bandwidth limits will kick in quickly so even if you have a very beefy machine bumping-it too high may not help.--You can always override these settings locally using command line as well:-gradlew -Ptests.jvms=N --max-workers=X--Test JVMS------------Test JVMs have their own set of arguments which can be customized. These are configured-separately from the gradle workers, for example:--tests.jvms=3-tests.heapsize=512m-tests.minheapsize=512m-tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1--Gradle Daemon----------------The gradle daemon is a background process that keeps an evaluated copy of the project-structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't-like the idea of having a (sizeable) background process running in the background,-disable it.--org.gradle.daemon=[true, false]-org.gradle.jvmargs=...+and save a project-local 'gradle.properties' file from a template,+with some default settings which you can modify.+See your 'gradle.properties' file for more details of those settings.diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f38985d58af..ecc780988a9 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -207,6 +207,8 @@ Build* SOLR-16625: Upgrade OWASP dependency check to 8.0.1 (Kevin Risden)+* SOLR-16641: Generate gradle.properties from gradlew, if absent (Colvin Cowie)+Other Changes---------------------* SOLR-16545: Upgrade Carrot2 to 4.5.0 (Dawid Weiss)