Apache Solr open-source search software
SOLR-15867 Bring in build stuff from lucene (#491)
Jan Høydahl32dd924b14ee
diff --git a/build.gradle b/build.gradleindex f4d380586fe..490c2f9c6f5 100644--- a/build.gradle+++ b/build.gradle@@ -127,7 +127,6 @@ apply from: file('gradle/java/jar-manifest.gradle')// Publishing and releasingapply from: file('gradle/maven/defaults-maven.gradle')-apply from: file('gradle/releasing.gradle')// IDE support, settings and specials.apply from: file('gradle/ide/intellij-idea.gradle')diff --git a/buildSrc/build.gradle b/buildSrc/build.gradleindex b1aee14c29d..91f88741322 100644--- a/buildSrc/build.gradle+++ b/buildSrc/build.gradle@@ -15,15 +15,24 @@* limitations under the License.*/+repositories {+ mavenCentral()+}+ext {// Minimum Java version required to compile buildSrc.minJavaVersion = JavaVersion.VERSION_11}+// Make sure the build environment is consistent.+apply from: file('../gradle/validation/check-environment.gradle')++// Load common buildSrc and script deps.+apply from: file("scriptDepVersions.gradle")+dependencies {implementation gradleApi()implementation localGroovy()-}-// Make sure the build environment is consistent.-apply from: file('../gradle/validation/check-environment.gradle')+ implementation "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"+}diff --git a/buildSrc/scriptDepVersions.gradle b/buildSrc/scriptDepVersions.gradlenew file mode 100644index 00000000000..5c27c518ba5--- /dev/null+++ b/buildSrc/scriptDepVersions.gradle@@ -0,0 +1,33 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Declare script dependency versions outside of palantir's+// version unification control. These are not our main dependencies+// but are reused in buildSrc and across applied scripts.++ext {+ // TODO: Check if Solr needs all of these+ scriptDepVersions = [+ "apache-rat": "0.11",+ "commons-codec": "1.13",+ "ecj": "3.27.0",+ "flexmark": "0.61.24",+ "javacc": "7.0.4",+ "jflex": "1.8.2",+ "jgit": "5.9.0.202009080501-r",+ ]+}diff --git a/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java b/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.javanew file mode 100644index 00000000000..0dab9dc7f05--- /dev/null+++ b/buildSrc/src/main/java/org/apache/lucene/gradle/Checksum.java@@ -0,0 +1,195 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * A task that generates SHA512 checksums. Cloned from:+ * https://github.com/gradle/gradle-checksum/blob/03351de/src/main/java/org/gradle/crypto/checksum/Checksum.java+ * with custom fixes to make the checksum palatable to shasum, see pending PR:+ * https://github.com/gradle/gradle-checksum/pull/4+ *+ * Original license ASL:+ * https://github.com/gradle/gradle-checksum/blob/03351de/LICENSE+ */++package org.apache.lucene.gradle;++import org.apache.commons.codec.digest.DigestUtils;+import org.gradle.api.DefaultTask;+import org.gradle.api.GradleException;+import org.gradle.api.file.FileCollection;+import org.gradle.api.file.FileType;+import org.gradle.api.tasks.Input;+import org.gradle.api.tasks.InputFiles;+import org.gradle.api.tasks.OutputDirectory;+import org.gradle.api.tasks.TaskAction;+import org.gradle.work.Incremental;+import org.gradle.work.InputChanges;++import java.io.File;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.util.Locale;++public class Checksum extends DefaultTask {+ private FileCollection files;+ private File outputDir;+ private Algorithm algorithm;++ public enum Algorithm {+ MD5(new DigestUtils(DigestUtils.getMd5Digest())),+ SHA256(new DigestUtils(DigestUtils.getSha256Digest())),+ SHA384(new DigestUtils(DigestUtils.getSha384Digest())),+ SHA512(new DigestUtils(DigestUtils.getSha512Digest()));++ private final DigestUtils hashFunction;++ Algorithm(DigestUtils hashFunction) {+ this.hashFunction = hashFunction;+ }++ public String getExtension() {+ return name().toLowerCase(Locale.ROOT);+ }+ }++ public Checksum() {+ outputDir = new File(getProject().getBuildDir(), "checksums");+ algorithm = Algorithm.SHA256;+ }++ @InputFiles+ @Incremental+ public FileCollection getFiles() {+ return files;+ }++ public void setFiles(FileCollection files) {+ this.files = files;+ }++ @Input+ public Algorithm getAlgorithm() {+ return algorithm;+ }++ public void setAlgorithm(Algorithm algorithm) {+ this.algorithm = algorithm;+ }++ @OutputDirectory+ public File getOutputDir() {+ return outputDir;+ }++ public void setOutputDir(File outputDir) {+ if (outputDir.exists() && !outputDir.isDirectory()) {+ throw new IllegalArgumentException("Output directory must be a directory.");+ }+ this.outputDir = outputDir;+ }++ @TaskAction+ public void generateChecksumFiles(InputChanges inputChanges) throws IOException {+ if (!getOutputDir().exists()) {+ if (!getOutputDir().mkdirs()) {+ throw new IOException("Could not create directory:" + getOutputDir());+ }+ }++ if (!inputChanges.isIncremental()) {+ getProject().delete(allPossibleChecksumFiles());+ }++ inputChanges+ .getFileChanges(getFiles())+ .forEach(+ fileChange -> {+ if (fileChange.getFileType() == FileType.DIRECTORY) {+ getProject()+ .getLogger()+ .warn("Checksums can't be applied to directories: " + fileChange.getFile());+ return;+ }++ File input = fileChange.getFile();+ switch (fileChange.getChangeType()) {+ case REMOVED:+ if (input.isFile()) {+ getProject().delete(outputFileFor(input));+ }+ break;++ case ADDED:+ case MODIFIED:+ input = fileChange.getFile();+ if (input.isFile()) {+ File checksumFile = outputFileFor(input);++ try {+ String checksum = algorithm.hashFunction.digestAsHex(input).trim();++ /*+ * https://man7.org/linux/man-pages/man1/sha1sum.1.html+ *+ * The default mode is to print a line with checksum, a space, a character+ * indicating input mode ('*' for binary, ' ' for text or where+ * binary is insignificant), and name for each FILE.+ */+ boolean binaryMode = true;++ Files.writeString(+ checksumFile.toPath(),+ String.format(+ Locale.ROOT,+ "%s %s%s",+ checksum,+ binaryMode ? "*" : " ",+ input.getName()),+ StandardCharsets.UTF_8);+ } catch (IOException e) {+ throw new GradleException("Trouble creating checksum: " + e.getMessage(), e);+ }+ }+ break;+ default:+ throw new RuntimeException();+ }+ });+ }++ private File outputFileFor(File inputFile) {+ return new File(getOutputDir(), inputFile.getName() + "." + algorithm.getExtension());+ }++ private FileCollection allPossibleChecksumFiles() {+ FileCollection possibleFiles = null;+ for (Algorithm algo : Algorithm.values()) {+ if (possibleFiles == null) {+ possibleFiles = filesFor(algo);+ } else {+ possibleFiles = possibleFiles.plus(filesFor(algo));+ }+ }+ return possibleFiles;+ }++ private FileCollection filesFor(final Algorithm algo) {+ return getProject()+ .fileTree(getOutputDir(), files -> files.include("**/*." + algo.toString().toLowerCase()));+ }+}diff --git a/gradle/documentation/changes-to-html.gradle b/gradle/documentation/changes-to-html.gradleindex e9692ab69d9..3b4ca69bf9d 100644--- a/gradle/documentation/changes-to-html.gradle+++ b/gradle/documentation/changes-to-html.gradle@@ -22,6 +22,18 @@ configure(project(':solr:documentation')) {siteDir = resourcesscript = file("${resources}/changes2html.pl")}++ // Make the rendered HTML of changes available as a separate+ // artifact for the distribution.+ configurations {+ changesHtml+ }++ artifacts {+ changesHtml changesToHtml.targetDir, {+ builtBy changesToHtml+ }+ }}// compile changes.txt into an html file@@ -44,7 +56,7 @@ class ChangesToHtmlTask extends DefaultTask {@OutputDirectoryfinal DirectoryProperty targetDir = project.objects.directoryProperty()- .fileProvider(project.providers.provider { project.file("${project.docroot}/changes") })+ .fileProvider(project.providers.provider { project.file("${project.docroot}/changes") })@Inputdef luceneDocUrl = "${-> project.luceneDocUrl }"@@ -72,11 +84,11 @@ class ChangesToHtmlTask extends DefaultTask {ignoreExitValue = trueargs += [- "-CSD",- script,- "${productName}",- versionsFile.toString(),- luceneDocUrl.concat('/') // slash required at end by perl script+ "-CSD",+ script,+ "${productName}",+ versionsFile.toString(),+ luceneDocUrl.concat('/') // slash required at end by perl script]}diff --git a/gradle/help.gradle b/gradle/help.gradleindex 7bbdaf5c207..3a354ed8c42 100644--- a/gradle/help.gradle+++ b/gradle/help.gradle@@ -30,7 +30,7 @@ configure(rootProject) {["Git", "help/git.txt", "Git assistance and guides."],["ValidateLogCalls", "help/validateLogCalls.txt", "How to use logging calls efficiently."],["IDEs", "help/IDEs.txt", "IDE support."],- ["GpgSigning", "help/gpgSigning.txt", "Signing artifacts with GPG."],+ ["Publishing", "help/publishing.txt", "Release publishing publishing, signing, etc."],["Docker", "solr/docker/gradle-help.txt", "Building Solr Docker images."],]diff --git a/gradle/releasing.gradle b/gradle/releasing.gradledeleted file mode 100644index b05b826fa40..00000000000--- a/gradle/releasing.gradle+++ /dev/null@@ -1,55 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import org.apache.commons.codec.digest.DigestUtils--// We're using commons-codec for computing checksums.-buildscript {- repositories {- mavenCentral()- }-- dependencies {- classpath "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"- }-}--allprojects {- plugins.withType(DistributionPlugin) {- def checksum = {- outputs.files.each { File file ->- String sha512 = new DigestUtils(DigestUtils.sha512Digest).digestAsHex(file).trim()- new File(file.parent, file.name + ".sha512").write(sha512 + " " + file.name, "UTF-8")- }- }-- distZip {- doLast checksum- }-- distTar {- compression = Compression.GZIP- doLast checksum- }-- installDist {- doLast {- logger.lifecycle "Distribution assembled under: ${destinationDir}"- }- }- }-}diff --git a/help/dependencies.txt b/help/dependencies.txtindex 2990db63c06..8cd78b72334 100644--- a/help/dependencies.txt+++ b/help/dependencies.txt@@ -13,8 +13,8 @@ https://docs.gradle.org/current/userguide/dependency_management_for_java_projecthttps://docs.gradle.org/current/userguide/java_library_plugin.html#sec:java_library_separationhttps://docs.gradle.org/current/userguide/java_plugin.html#sec:java_plugin_and_dependency_management-For the needs of Solr we will typically focus on three-configurations and attach project dependencies to them:+Solr typically uses three configurations and attach project+dependencies to them:api - makes a dependency available for main classes, tests and anyother modules importing the project (exportable dependency),diff --git a/help/gpgSigning.txt b/help/gpgSigning.txtdeleted file mode 100644index 80e45f0e1d2..00000000000--- a/help/gpgSigning.txt+++ /dev/null@@ -1,70 +0,0 @@-GPG Signing-===========--GPG Signing of distribution files (typically by a release manager) is done with the 'signDist' command.--The only required configuration property Gradle needs is the 'signing.gnupg.keyName' (aka: the fingerprint) of-the key you wish to use:--./gradlew signDist -Psigning.gnupg.keyName=YOUR_KEY_FINGERPRINT--By default when you run this command, Gradle will delegate to the `gpg2` command for managing the signing of each file, which (should)-in turn use the `gpg-agent` to prompt you for your secret key only as needed using a dialog box specific to your operating system and/or-`gpg-agent` preferences.--You may wish to put the `signing.gnupg.keyName` in your `~/.gradle/gradle.properties` so it is set automatically any time you use gradle---Additional Configuration---------------------------The following additional properties -- specified either on the command line via `-P...` or in your `~/.gradle/gradle.properties` may be-useful/necessary in your system:--signing.gnupg.useLegacyGpg=true # Changes the default executable from `gpg2` to `gpg` and explicitly sets `--use-agent`-signing.gnupg.executable=gpg # Allows explicit control over what command executable used (ex: `gpg2`, `gpg`, `gpg.exe`, etc...)-signing.gnupg.homeDir=/tmp/gnupg-home # overrides GnuPG's default home directory (ex: `~/.gnupg/`)-signing.gnupg.optionsFile=/tmp/gnupg-home/my.conf # overrides GnuPG's default configuration file-signing.gnupg.passphrase=... # Provide your passphrase to gradle to hand off to gpg. *NOT RECOMMENDED*, see below.---Notes About Error Messages------------------------------### `gpg: signing failed: Inappropriate ioctl for device`--This typically happens if your `gpg-agent` is configured (either globally for your operating system, or personally in your-`~/.gnupg/gpg-agent.conf`) to use a `pinentry` command which depends on using the same `tty` as the `gpg` command (ex: `pinentry-curses`,-or `pinentry-tty`, etc...).--`tty` based `pinentry` implementations do not work when Gradle's `SigningPlugin` is attempting to invoke `gpg` -- among other problems:-Gradle is multi-threaded, and we sign multiple artifacts by default; so even if the `SigningPlugin` didn't automatically force `--no-tty` when-running `gpg` you could easily run into problems where a second `pinentry` process wanted to read from the same `tty` in the middle of you-typing in your passphrase to the first process.--Developers are encouraged to configure a *non* `tty` based `pinentry` (ex: `pinentry-gnome`, `pinentry-x11`, `pinentry-qt`, `pinentry-mac`,-`pinentry-wsl-ps1`, etc...) either globally in your operating system, or personally in your `~/.gnupg/gpg-agent.conf`, or in a new-`gpg-agent.conf` file a new GnuPG configuration directory (containing a copy of your private keys) that you direct gradle to via-`signing.gnupg.homeDir`--If none of these options are viable for you, then as a last resort you may wish to consider using the `signing.gnupg.passphrase=...` property.-This will expose your secret passphrase to the Gradle process, which will then pass it directly to each `gpg-agent` instance using-`--pinentry-mode=loopback`.---### `gpg: signing failed: No such file or directory`--This may mean that there is a problem preventing `gpg` from communicating correctly with the `gpg-agent` (and/or invoking your `pinentry`-program) that is independent of gradle. Try running `pkill gpg-agent` and then retrying your `./gradlew` command---### `No value has been specified for property 'signatory.keyId'.`--Do not bother ever attempting to set a command line (or gradle.properties) property named `signatory.keyId`. This is evidently the-name of an internal property that the gradle `SigningPlugin` expects the `GnupgSignatory` plugin we use to provide -- which it does-as long as you have specified a valid value for `signing.gnupg.keyName`--If you see this error, it means you did not properly set `signing.gnupg.keyName` _AND_ you invoked a task which is attempting to use-the `SigningPlugin`, but does not depend on the custom `failUnlessGpgKeyProperty` to report the error correctly. Please file a Jira-noting what `./gradlew` command you attempted to run so we can fix it's dependencies, and try again after setting `signing.gnupg.keyName`.diff --git a/help/publishing.txt b/help/publishing.txtnew file mode 100644index 00000000000..5b8bc5dd275--- /dev/null+++ b/help/publishing.txt@@ -0,0 +1,138 @@+Distribution and artifact publishing+====================================+++See all distribution-related tasks by running:+gradlew tasks --group distribution+++Maven+-----++To publish Solr Maven artifacts to a local ~/.m2 repository, run:++gradlew mavenToLocalRepo++To generate a local build/maven-local folder, run:++gradlew mavenToLocalFolder++To publish Solr Maven artifacts to Apache repositories+(CI or release manager's job, typically!), run:++gradlew mavenToApacheSnapshots -PasfNexusUsername= -PasfNexusPassword=+gradlew mavenToApacheReleases -PasfNexusUsername= -PasfNexusPassword= [optional signing options]++See artifact signing section below if you plan to use mavenToApacheReleases.++It is a good idea to avoid passing passwords on command line. CI jobs have+these properties saved in ~/.gradle/gradle.properties - this way they+are read automatically.++Apache Releases repository will not accept snapshots.+++Release (distribution) artifacts+--------------------------------++To collect all release artifacts, and optionally sign them, run:++gradlew assembleRelease [optional signing options]++All distribution artifacts will be placed under:++solr/distribution/build/release++Artifact signing is optional (but required if you're really making a release).+++Artifact signing+----------------++Certain tasks may optionally sign artifacts or require artifacts to be signed:++ assembleRelease+ mavenToApacheReleases++Signing can be enabled by adding the "-Psign" option, for example:++gradlew assembleRelease mavenToApacheReleases -Psign++By default gradle uses a Java-based implementation of PGP for signing, which requieres+several "signing.*" properties via either ~/.gradle/gradle.properties or command-line options:++https://docs.gradle.org/current/userguide/signing_plugin.html#sec:signatory_credentials++An example full command-line that assembles signed artifacts could look like this:++gradlew assembleRelease mavenToApacheReleases -Psign -Psigning.keyId=... -Psigning.password=... -Psigning.secretKeyRingFile=...++The keyId is the last 8 digits of your key (gpg -k will print your keys). Gradle documentation has more options+of secure passing of private key information and passwords.+++Artifact signing using an external GPG with GPG Agent+-----------------------------------------------------++You can use an external GPG command to deal with signing artifacts, with out needing to give gradle your passphrase,+by adding a "-PuseGpg" option, but this changes the properties you must specify:++For gpg2:+gradlew [tasks] -Psign -PuseGpg -Psigning.gnupg.keyName=...++For gpg:+gradlew [tasks] -Psign -PuseGpg -Psigning.gnupg.keyName=... -Psigning.gnupg.useLegacyGpg=true++The keyName is the last 8 digits of your key (gpg -k will print your keys).++There are additional (optional) "signing.gnupg.*" properties which exist that may be useful/necessary in your system:++signing.gnupg.useLegacyGpg=true # Changes the default executable from `gpg2` to `gpg` and explicitly sets `--use-agent`+signing.gnupg.executable=gpg # Allows explicit control over what command executable used (ex: `gpg2`, `gpg`, `gpg.exe`, etc...)+signing.gnupg.homeDir=/tmp/gnupg-home # overrides GnuPG's default home directory (ex: `~/.gnupg/`)+signing.gnupg.optionsFile=/tmp/gnupg-home/my.conf # overrides GnuPG's default configuration file+signing.gnupg.passphrase=... # Provide your passphrase to gradle to hand off to gpg. *NOT RECOMMENDED*, see below.++If in doubt, consult gradle's signing plugin documentation:+https://docs.gradle.org/current/userguide/signing_plugin.html#sec:using_gpg_agent++"signing.gnupg.passphrase" is not recommended because there is no advantage to using an external GPG process if you use it. If you+are comfortable giving gradle your passphrase, then there is no reason to use an external GPG process via '-PuseGpg'. Just use the+"signing.*" options described previously to let gradle deal with your key directly.++Because of how Gradle's signing plugin invokes GPG, using an external GPG process *only* works if your GPG configuration uses a+GPG agent (required by gpg2) and if the "pinentry" for your GPG agent does not require access to the tty to prompt you for a password.++If you the following command fails with your GPG configuration, you can not use an external GPG process with gradle:++echo foo | gpg --batch --no-tty --armor --detach-sign --use-agent --local-user YOUR_KEY_NAME+++Notes About GPG Error Messages+------------------------------++### `gpg: signing failed: Inappropriate ioctl for device` or `Invalid IPC response`++This typically happens if your `gpg-agent` is configured (either globally for your operating system, or personally in your+`~/.gnupg/gpg-agent.conf`) to use a `pinentry` command which depends on using the same `tty` as the `gpg` command (ex: `pinentry-curses`,+or `pinentry-tty`, etc...).++`tty` based `pinentry` implementations do not work when Gradle's signing plugin is attempting to invoke `gpg` -- among other problems:+Gradle is multi-threaded and we sign multiple artifacts by default. Even if you use "--max-workers 1" to force single-threaded execution,+the signing plugin invokes gpg with `--batch --no-tty`, making it impossible for gpg (or a tty based pinentry) to prompt you for your passphrase+in the same terminal where you run Gradle.++Developers are encouraged to configure a *non* `tty` based `pinentry` (ex: `pinentry-gnome`, `pinentry-x11`, `pinentry-qt`, `pinentry-mac`,+`pinentry-wsl-ps1`, etc...) either globally in your operating system, or personally in your `~/.gnupg/gpg-agent.conf`, or in a new+`gpg-agent.conf` file a new GnuPG configuration directory (containing a copy of your private keys) that you direct gradle to via+`signing.gnupg.homeDir`++If this is not possible, then you should avoid using an external GPG process, and use the default (pure java) Artifact signing support+++### `gpg: signing failed: No such file or directory`++This may mean that there is a problem preventing `gpg` from communicating correctly with the `gpg-agent` (and/or invoking your `pinentry`+program) that is independent of gradle. Try running `pkill gpg-agent` and then retrying your `./gradlew` command++diff --git a/help/workflow.txt b/help/workflow.txtindex 2b22c3c8618..c59d77dab68 100644--- a/help/workflow.txt+++ b/help/workflow.txt@@ -27,11 +27,22 @@ local maven repository for inspection:gradlew mavenLocalls -R build/maven-local/-Put together Solr distribution:+Put together a local Solr binary "distribution" folder:gradlew -p solr/packaging assemble-ls solr/packaging/build/distributions/solr-* # release archivesls solr/packaging/build/solr-* # expanded directory+For quick local development+gradlew -p solr/packaging dev+ls solr/packaging/build/dev # expanded directory++Generate the release tar and zip archives (see publishing.txt for details)+gradlew -p solr/distribution assembleRelease+ls solr/distribution/build/release # release archives++Build a docker image from the local repository (see docker/gradle-help.txt for more)+gradlew dockerBuild dockerTag+docker run --rm -p 8983:8983 apache/solr:9.0.0-SNAPSHOT+Other validation and checks===========================diff --git a/settings.gradle b/settings.gradleindex a04cd7c591b..88eab3965b3 100644--- a/settings.gradle+++ b/settings.gradle@@ -47,6 +47,7 @@ include "solr:solr-ref-guide"include "solr:example"include "solr:documentation"include "solr:packaging"+include "solr:distribution"include "solr:docker"// Configures development for joint Lucene/ Solr composite build.diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 62cf2f1caad..11a0253fed8 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -230,6 +230,8 @@ Build* SOLR-15793: Pin http_parser.rb to specific version to allow Solr Ref Guide on Jekyll to run (Eric Pugh)+* SOLR-15867: Make the gradle build ready for 9.0 release (janhoy, Houston Putman, Dawid Weiss)+Other Changes----------------------* SOLR-15603: Add an option to activate Gradle build cache, build task cleanups (Alexis Tual, Dawid Weiss)diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradlenew file mode 100644index 00000000000..a586b09c172--- /dev/null+++ b/solr/distribution/build.gradle@@ -0,0 +1,160 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import org.apache.lucene.gradle.Checksum++import java.nio.charset.StandardCharsets+import java.nio.file.Files++plugins {+ id 'signing'+}++// This project puts together the Solr "distribution", assembling bits and pieces+// from across the project structure into release artifacts.++ext {+ releaseDir = file("${buildDir}/release")+ withSignedArtifacts = { ->+ def propValue = propertyOrDefault("sign", null)+ // Allow -Psign to work as a shorthand for -Psign=true+ return propValue != null && (propValue.isBlank() || Boolean.parseBoolean(propValue))+ }.call()+ useGpgForSigning = { ->+ def propValue = propertyOrDefault("useGpg", null)+ // Allow -PuseGpg to work as a shorthand for -PuseGpg=true+ return propValue != null && (propValue.isBlank() || Boolean.parseBoolean(propValue))+ }.call()+}++if (project.ext.useGpgForSigning) {+ signing {+ useGpgCmd()+ }+}++// Prepare the "source" distribution artifact.+apply from: buildscript.sourceFile.toPath().resolveSibling("source-release.gradle")+++// Set up the HTML-rendered "changes" distribution artifact by linking to documentation's output.+configurations {+ changesHtml+ docker+}++dependencies {+ changesHtml project(path: ":solr:documentation", configuration: "changesHtml")+ docker project(path: ':solr:docker', configuration: project.ext.withSignedArtifacts ? 'packagingOfficial' : 'packaging')+}++def distTarTask = rootProject.getTasksByName("distTar", true)[0]+def distZipTask = rootProject.getTasksByName("distZip", true)[0]++// Compute checksums for release archives.+task computeChecksums(type: Checksum) {+ algorithm = Checksum.Algorithm.SHA512++ files = objects.fileCollection()+ [+ tasks.assembleSourceTgz,+ distTarTask,+ distZipTask,+ ].each { dep ->+ dependsOn dep+ files += dep.outputs.files+ }++ outputDir = file("${buildDir}/checksums")+}++task signBinaryTgz(type: Sign) {+ sign distTarTask+}+task signBinaryZip(type: Sign) {+ sign distZipTask+}+task signSourceTgz(type: Sign) {+ // The source tgz is not an archive task so be explicit about the outputs to sign.+ dependsOn tasks.assembleSourceTgz+ sign tasks.assembleSourceTgz.destination+}++task signReleaseArchives(type: Sync) {+ from tasks.signBinaryTgz+ from tasks.signBinaryZip+ from tasks.signSourceTgz++ into "${buildDir}/signatures"+}++task prepareGitRev() {+ dependsOn ":gitStatus"++ ext.outputFile = file("${buildDir}/.gitrev")++ outputs.file(ext.outputFile)+ inputs.property("gitrev", provider { -> rootProject.ext.gitRev })++ doFirst {+ Files.writeString(ext.outputFile.toPath(), rootProject.ext.gitRev, StandardCharsets.UTF_8)+ }+}+++// Assemble everything needed in the release folder structure.+task assembleRelease(type: Sync) {+ description "Assemble all Solr artifacts for a release."+ dependsOn ":mavenToLocalFolder"++ from(configurations.changesHtml, {+ into "changes"+ })++ from(configurations.docker, {+ include 'Dockerfile.*'+ into "docker"+ })++ from(rootProject.mavenLocalDir, {+ into "maven"+ })++ from tasks.prepareGitRev+ from tasks.assembleSourceTgz+ from distTarTask+ from distZipTask++ from tasks.computeChecksums++ // Conditionally, attach signatures of all the release archives.+ if (project.ext.withSignedArtifacts) {+ from tasks.signReleaseArchives+ }++ into releaseDir+}+++// Add the description and task group to some of the tasks that make+// sense at the user-level help.+tasks.matching {it.name in [+ "assembleSourceTgz",+ "assembleRelease",+]}.all {+ group "distribution"+}diff --git a/solr/distribution/source-release.gradle b/solr/distribution/source-release.gradlenew file mode 100644index 00000000000..9e0c11b0dcb--- /dev/null+++ b/solr/distribution/source-release.gradle@@ -0,0 +1,50 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+++// Prepare the "source" distribution artifact. We use raw git export, no additional complexity needed.++configure(project(":solr:distribution")) {+ task assembleSourceTgz() {+ description "Assemble source Solr artifact as a .tgz file."++ // Make sure no unstaged/ dirty files are present.+ dependsOn ":gitStatus", ":checkWorkingCopyClean"++ ext {+ destination = file("${buildDir}/packages/solr-${version}-src.tgz")+ }++ inputs.property("git-revision", { -> rootProject.ext.gitRev })+ outputs.file destination++ doFirst {+ quietExec {+ executable = project.externalTool("git")+ workingDir = project.rootDir++ args += [+ "archive",+ "--format", "tgz",+ "--prefix", "solr-${version}/",+ "--output", destination,+ "HEAD"+ ]+ }+ }+ }+}diff --git a/solr/docker/build.gradle b/solr/docker/build.gradleindex c2ed0140dd2..92cbb260674 100644--- a/solr/docker/build.gradle+++ b/solr/docker/build.gradle@@ -29,7 +29,7 @@ def dockerImageTag = propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKERdef dockerImageName = propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}")def baseDockerImage = propertyOrEnvOrDefault("solr.docker.baseImage", "SOLR_DOCKER_BASE_IMAGE", 'openjdk:11-jre-slim')-def releaseGpgFingerprint = propertyOrDefault('signing.gnupg.keyName','');+def releaseGpgFingerprint = propertyOrDefault('signing.gnupg.keyName',propertyOrDefault('signing.keyId',''));// Build directory locationsdef imageIdFile = "$buildDir/image-id"@@ -41,6 +41,9 @@ configurations {packaging {canBeResolved = true}+ packagingOfficial {+ canBeResolved = true+ }solrTgz {canBeConsumed = falsecanBeResolved = true@@ -59,6 +62,7 @@ configurations {}ext {+ dockerfilesDirPath = "${buildDir}/dockerfiles"packagingDir = file("${buildDir}/packaging")}@@ -67,7 +71,11 @@ dependencies {builtBy 'assemblePackaging'}- solrTgz project(path: ":solr:packaging", configuration: 'solrTgz')+ packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official") {+ builtBy 'createDockerfileOfficial'+ }++ solrTgz project(path: ":solr:packaging", configuration: "solrTgz")solrTgzSignature project(path: ":solr:packaging", configuration: 'solrTgzSignature')dockerImage files(imageIdFile) {@@ -98,7 +106,7 @@ task assemblePackaging(type: Sync) {from(projectDir, {include "scripts/**"})- from(buildDir, {+ from(dockerfilesDirPath, {include 'Dockerfile.local'})into packagingDir@@ -271,7 +279,7 @@ The final Dockerfiles are merely the snippet headers combined with the snippet b*/[ dfLocalDetails, dfOfficialDetails ].each{ details ->def fileName = "Dockerfile.${ -> details.name.toLowerCase(Locale.ROOT) }"- def outFile = file("$buildDir/${fileName}")+ def outFile = file("$dockerfilesDirPath/${fileName}")def props = [// Values defined here should be common (and consistent) across both Dockerfiles@@ -327,7 +335,7 @@ tasks.createDockerfileOfficial.dependsOn configurations.solrTgz // to lazy compuif (''.equals(releaseGpgFingerprint)) {gradle.taskGraph.whenReady { graph ->if ( graph.hasTask(createDockerfileOfficial) ) {- throw new GradleException("No GPG keyName found, please see help/gpgSigning.txt (GPG key is neccessary to create Dockerfile.official)")+ throw new GradleException("No GPG keyName found, please see help/publishing.txt (GPG key is neccessary to create Dockerfile.official)")}}}@@ -338,10 +346,10 @@ task testBuildDockerfileOfficial(type: Copy) {dependsOn createDockerfileOfficialdependsOn configurations.solrTgzdependsOn configurations.solrTgzSignature-+def mockHttpdHome = file("$smokeTestOfficial/mock-httpd-home");- inputs.file("$buildDir/Dockerfile.official")+ inputs.file("$dockerfilesDirPath/Dockerfile.official")outputs.dir(mockHttpdHome)outputs.file(imageIdFileOfficial)@@ -383,7 +391,7 @@ task testBuildDockerfileOfficial(type: Copy) {// *NOW* we can actually run our docker build command...logger.lifecycle('Running docker build on Dockerfile.official...');exec {- standardInput = file("${buildDir}/Dockerfile.official").newDataInputStream()+ standardInput = file("${dockerfilesDirPath}/Dockerfile.official").newDataInputStream()commandLine 'docker', 'build','--add-host', "mock-solr-dl-server:${mockServerIp}",'--no-cache', // force fresh downloads from our current networkdiff --git a/solr/docker/gradle-help.txt b/solr/docker/gradle-help.txtindex 765fd4483bf..83cc439831a 100644--- a/solr/docker/gradle-help.txt+++ b/solr/docker/gradle-help.txt@@ -93,7 +93,7 @@ All users should build custom images using the instructions above.NOTE: All gradle commands for the Official Dockerfile below require the Solr artifacts to be signed with a GPG Key.For necessary inputs and properties, please refer to:-gradlew helpGpgSigning+gradlew helpPublishingYou can use the following command to build an official Solr Dockerfile.The Dockerfile will be created at: solr/docker/build/Dockerfile.officialdiff --git a/solr/docker/templates/Dockerfile.body.template b/solr/docker/templates/Dockerfile.body.templateindex 4aa7d3ca650..110a7a93b32 100644--- a/solr/docker/templates/Dockerfile.body.template+++ b/solr/docker/templates/Dockerfile.body.template@@ -30,7 +30,7 @@# TODO; arguably these permissions should have been set correctly previously in the TARRUN set -ex; \(cd /opt; ln -s solr-*/ solr); \- rm -Rf /opt/solr/docs /opt/solr/docker/Dockerfile* /opt/solr/dist/solr-solrj-*.jar /opt/solr/dist/solrj-lib /opt/solr/dist/solr-core-*.jar; \+ rm -Rf /opt/solr/docs /opt/solr/dist/solr-solrj-*.jar /opt/solr/dist/solrj-lib /opt/solr/dist/solr-core-*.jar; \find /opt/solr/ -type d -print0 | xargs -0 chmod 0755; \find /opt/solr/ -type f -print0 | xargs -0 chmod 0644; \chmod -R 0755 /opt/solr/docker/scripts /opt/solr/bin /opt/solr/contrib/prometheus-exporter/bin/solr-exporter /opt/solr/server/scripts/cloud-scriptsdiff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradleindex 8f98c67d6a5..e2f86d1114b 100644--- a/solr/packaging/build.gradle+++ b/solr/packaging/build.gradle@@ -21,7 +21,6 @@plugins {id 'base'id 'distribution'- id 'signing'}description = 'Solr distribution packaging'@@ -81,6 +80,10 @@ dependencies {docs project(path: ':solr:documentation', configuration: 'minimalSite')docker project(path: ':solr:docker', configuration: 'packaging')++ solrTgzSignature files("$buildDir/distributions/solr-${version}.tgz.asc") {+ builtBy ":solr:distribution:signBinaryTgz"+ }}distributions {@@ -112,14 +115,6 @@ distributions {include "README.md"})- // Should we include Lucene changes?- /*- from(project(":lucene").projectDir, {- include "CHANGES.txt"- rename { file -> 'LUCENE_CHANGES.txt' }- })- */-from(configurations.contrib, {into "contrib"})@@ -164,48 +159,8 @@ task dev(type: Copy) {into devDir}-assemble.dependsOn installDist---// NOTE: we don't use the convinence DSL of the 'signing' extension to define our 'Sign' tasks because-// that (by default) adds our signature files to the 'archives' configuration -- which is what-// assemble & installDist try to copy/sync, so they wouldn't work w/o GPG installed (which would be bad).-//-// We also want to hook in our own property check dependency since the default error message from Sign task-// refers to the wrong (internal only) property name ("signatory.keyId")-signing {- useGpgCmd() // so gpg-agent can be used-}-task failUnlessGpgKeyProperty {- // placeholder that can be depended on by any task needing GPG key which will 'fail fast' if it's not set.- def propName = 'signing.gnupg.keyName'-- // This explicitly checks the taskGraph (instead of a simple 'doFirst') so it can fail the user's gradle- // invocation immediately before any unrelated build tasks may run in parallel- if ( ! project.hasProperty(propName) ) {- gradle.taskGraph.whenReady { graph ->- if ( graph.hasTask(failUnlessGpgKeyProperty) ) {- // TODO: it would be really nice if taskGraph was an actual graph and we could report what tasks in (transitive) depend on us- throw new GradleException("'$propName' property must be set for GPG signing, please see help/gpgSigning.txt")- }- }- }-}-task signDistTar(type: Sign) {- dependsOn failUnlessGpgKeyProperty- sign configurations.solrTgz-}-dependencies {- solrTgzSignature files(tasks.signDistTar.signatureFiles.singleFile) {- builtBy signDistTar- }-}-task signDistZip(type: Sign) {- dependsOn failUnlessGpgKeyProperty- sign configurations.solrZip-}-task signDist {- group = 'Distribution'- description = 'GPG Signs the main distributions'- dependsOn signDistTar, signDistZip+distTar {+ compression = Compression.GZIP}++assemble.dependsOn installDist\ No newline at end of file