Apache Solr open-source search software
SOLR-16748: Official Dockerfile cleanup Args & GPG/SHA (#1563)
Houston Putman69871072e03c
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.pyindex 6c08df84b70..876aadd509f 100755--- a/dev-tools/scripts/buildAndPushRelease.py+++ b/dev-tools/scripts/buildAndPushRelease.py@@ -217,7 +217,7 @@ def pushLocal(version, root, rcNum, localDir):rev = open('%s/solr/distribution/build/release/.gitrev' % root, encoding='UTF-8').read()dir = 'solr-%s-RC%d-rev-%s' % (version, rcNum, rev)- os.makedirs('%s/%s/solr' % (localDir, dir))+ os.makedirs('%s/%s/solr/%s' % (localDir, dir, version))print(' Solr')solr_dist_dir = '%s/solr/distribution/build/release' % rootos.chdir(solr_dist_dir)@@ -227,7 +227,7 @@ def pushLocal(version, root, rcNum, localDir):run('tar cf solr.tar *')print(' extract...')- os.chdir('%s/%s/solr' % (localDir, dir))+ os.chdir('%s/%s/solr/%s' % (localDir, dir, version))run('tar xf "%s/solr.tar"' % solr_dist_dir)os.remove('%s/solr.tar' % solr_dist_dir)os.chdir('..')diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yamlindex c5c70e5c4a8..3e57d5ae467 100644--- a/dev-tools/scripts/releaseWizard.yaml+++ b/dev-tools/scripts/releaseWizard.yaml@@ -863,14 +863,13 @@ groups:You can build a release-candidate of the official docker images (full & slim) using the following command:- DIST_BASE={{ dist_url_base }} && \- RC_FOLDER={{ release_candidate_rc_folder }} && \- docker build $DIST_BASE/$RC_FOLDER/solr/docker/Dockerfile.official-full \- --build-arg SOLR_DOWNLOAD_URL=$DIST_BASE/$RC_FOLDER/solr/solr-{{ release_version }}.tgz \- -t solr-rc:{{ release_version }}-{{ rc_number }}} && \- docker build $DIST_BASE/$RC_FOLDER/solr/docker/Dockerfile.official-slim \- --build-arg SOLR_DOWNLOAD_URL=$DIST_BASE/$RC_FOLDER/solr/solr-{{ release_version }}-slim.tgz \- -t solr-rc:{{ release_version }}-{{ rc_number }}-slim+ SOLR_DOWNLOAD_SERVER={{ release_candidate_location }}/solr && \+ docker build $SOLR_DOWNLOAD_SERVER/{{ release_version }}/docker/Dockerfile.official-full \+ --build-arg SOLR_DOWNLOAD_SERVER=$DIST_BASE/$RC_FOLDER/solr \+ -t solr-rc:{{ release_version }}-{{ rc_number }}} && \+ docker build $SOLR_DOWNLOAD_SERVER/{{ release_version }}/docker/Dockerfile.official-slim \+ --build-arg SOLR_DOWNLOAD_SERVER=$DIST_BASE/$RC_FOLDER/solr \+ -t solr-rc:{{ release_version }}-{{ rc_number }}-slimThe vote will be open for at least 72 hours i.e. until {{ vote_close }}.@@ -1002,7 +1001,7 @@ groups:commands_text: This will remove maven and docker artifacts so they do not end up in the Distribution Directorycommands:- !Command- cmd: svn rm -m "Delete the solr maven and docker artifacts" {{ dist_stage_url }}/solr/maven {{ dist_stage_url }}/solr/docker+ cmd: svn rm -m "Delete the solr maven and docker artifacts" {{ dist_stage_url }}/solr/{{ release_version }}/maven {{ dist_stage_url }}/solr/{{ release_version }}/dockerlogfile: svn_rm_mvn_solr.logtee: true- !Todo@@ -1019,7 +1018,7 @@ groups:commands_text: This will move the new release artifacts from staging repo to the release repocommands:- !Command- cmd: svn move -m "Move Solr {{ release_version }} RC{{ rc_number }} to release repo" {{ dist_stage_url }}/solr {{ dist_release_url }}/solr/{{ release_version }}+ cmd: svn move -m "Move Solr {{ release_version }} RC{{ rc_number }} to release repo" {{ dist_stage_url }}/solr/{{ release_version }} {{ dist_release_url }}/solr/{{ release_version }}logfile: svn_mv_solr.logtee: true- !Command@@ -1058,12 +1057,12 @@ groups:comment: Checkout the most recent main versionlogfile: solr_docker_git_checkout.log- !Command- cmd: "mkdir -p {{ docker_version }} && cp {{ [dist_file_path, dist_folder, 'solr', 'docker', 'Dockerfile.official-full'] | path_join }} {{ docker_version }}/Dockerfile"+ cmd: "mkdir -p {{ docker_version }} && cp {{ [dist_file_path, dist_folder, 'solr', release_version, 'docker', 'Dockerfile.official-full'] | path_join }} {{ docker_version }}/Dockerfile"cwd: solr-dockercomment: Add the new full distribution Dockerfile to the repologfile: solr_docker_add_dockerfile_full.log- !Command- cmd: "mkdir -p {{ docker_version }} && cp {{ [dist_file_path, dist_folder, 'solr', 'docker', 'Dockerfile.official-slim'] | path_join }} {{ docker_version }}-slim/Dockerfile"+ cmd: "mkdir -p {{ docker_version }}-slim && cp {{ [dist_file_path, dist_folder, 'solr', release_version, 'docker', 'Dockerfile.official-slim'] | path_join }} {{ docker_version }}-slim/Dockerfile"cwd: solr-dockercomment: Add the new slim distribution Dockerfile to the repologfile: solr_docker_add_dockerfile_slim.log@@ -1096,7 +1095,7 @@ groups:commands_text: In the source checkout do the following (note that this step will prompt you for your Apache LDAP credentials)commands:- !Command- cmd: bash dev-tools/scripts/upload-maven.sh -u {{ gpg.apache_id }} -v {{ release_version }} -c {{ git_sha }} -d "{{ [dist_file_path, dist_folder, 'solr', 'maven'] | path_join }}"+ cmd: bash dev-tools/scripts/upload-maven.sh -u {{ gpg.apache_id }} -v {{ release_version }} -c {{ git_sha }} -d "{{ [dist_file_path, dist_folder, 'solr', release_version, 'maven'] | path_join }}"tee: truelogfile: publish_solr_maven.logpost_description: The artifacts are not published yet, please proceed with the next step to actually publish!diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.pyindex 01e4a839429..50b517aae59 100755--- a/dev-tools/scripts/smokeTestRelease.py+++ b/dev-tools/scripts/smokeTestRelease.py@@ -756,7 +756,7 @@ def testSolrExample(binaryDistPath, javaPath):run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log')print(' run query...')s = load('http://localhost:8983/solr/techproducts/select/?q=video')- if s.find('"numFound":3') == -1:+ if s.find('"numFound":3,') == -1:print('FAILED: response is:\n%s' % s)raise RuntimeError('query on solr example instance failed')s = load('http://localhost:8983/api/cores')@@ -1130,11 +1130,14 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,else:# An ordinary release has a 'solr' sub folderfor text, subURL in getDirEntries(baseURL):- if text.lower() == 'solr/':+ if text.lower() == "solr/":+ solrPath = subURL+ for text, subURL in getDirEntries(solrPath):+ if text.lower() == version + "/":solrPath = subURLif solrPath is None:- raise RuntimeError('could not find solr subdir')+ raise RuntimeError("could not find solr/%s/ subdir" % version)print()print('Get KEYS...')diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 400ee744dc5..e472e4b7cf2 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -180,6 +180,10 @@ Improvements* SOLR-16820: Align allowed collection names and the validation of them in the CreateCollectionAPI and the PackageTool. (Will White via Eric Pugh)+* SOLR-16748: The Official Solr Dockerfile has been cleaned up. The SOLR_DOWNLOAD_SERVER arg is now the only way to+ specify where to download Solr from. If the url includes "apache.org", then GPG checks will be computed, otherwise+ the GPG checks will be skipped. (Houston Putman)+Optimizations---------------------diff --git a/solr/docker/build.gradle b/solr/docker/build.gradleindex a84e4ec245a..a2c6194e820 100644--- a/solr/docker/build.gradle+++ b/solr/docker/build.gradle@@ -455,8 +455,9 @@ if (''.equals(releaseGpgFingerprint)) {'--cidfile', mockServerIdFile,'--rm','-d',- '-v', "${mockHttpdHome.absoluteFile}:/data",+ '-v', "${mockHttpdHome.absoluteFile}:/data/${-> project.version}",'-w', '/data',+ '-p', '9876:9876','python:3-alpine', 'python', '-m', 'http.server', '9876'}try {@@ -473,11 +474,10 @@ if (''.equals(releaseGpgFingerprint)) {exec {standardInput = file("${dockerfilesDirPath}/Dockerfile.official-${lowerVariant}").newDataInputStream()commandLine 'docker', 'build',- '--add-host', "mock-solr-dl-server:${mockServerIp}",+ '--add-host', "mock-downloads.apache.org:${mockServerIp}",'--no-cache', // force fresh downloads from our current network"--iidfile", imageIdFileOfficial(variant),- '--build-arg', "SOLR_CLOSER_URL=http://mock-solr-dl-server:9876/solr-${-> project.version}${distToSuffix(variant)}.tgz",- '--build-arg', "SOLR_ARCHIVE_URL=http://mock-solr-dl-server:9876/solr-${-> project.version}${distToSuffix(variant)}.tgz",+ '--build-arg', "SOLR_DOWNLOAD_SERVER=http://mock-downloads.apache.org:9876",'--tag', officialDockerImageName(variant),'-'}diff --git a/solr/docker/templates/Dockerfile.official.header.template b/solr/docker/templates/Dockerfile.official.header.templateindex 3a69f3e756d..add0d923a29 100644--- a/solr/docker/templates/Dockerfile.official.header.template+++ b/solr/docker/templates/Dockerfile.official.header.template@@ -21,72 +21,62 @@FROM _REPLACE_BASE_IMAGE_-# TODO: remove things that exist solely for downstream specialization since Dockerfile.local now exists for that-ARG SOLR_VERSION="_REPLACE_SOLR_VERSION_"# empty for the full distribution, "-slim" for the slim distributionARG SOLR_DIST="_REPLACE_SOLR_DIST_"ARG SOLR_SHA512="_REPLACE_SOLR_TGZ_SHA_"ARG SOLR_KEYS="_REPLACE_RELEASE_MANAGER_GPG_FINGERPRINT_"-# If specified, this will override SOLR_DOWNLOAD_SERVER and all ASF mirrors. Typically used downstream for custom builds-ARG SOLR_DOWNLOAD_URL-# TODO: That comment isn't strictly true, if SOLR_DOWNLOAD_URL fails, other mirrors will be attempted-# TODO: see patch in SOLR-15250 for some example ideas on fixing this to be more strict--# Override the default solr download location with a prefered mirror, e.g.:+# Override the default solr download location with a preferred mirror, e.g.:# docker build -t mine --build-arg SOLR_DOWNLOAD_SERVER=https://downloads.apache.org/solr/solr .-ARG SOLR_DOWNLOAD_SERVER--# These should never be overridden except for the purposes of testing the Dockerfile before release-ARG SOLR_CLOSER_URL="http://www.apache.org/dyn/closer.lua?action=download&filename=/solr/solr/$SOLR_VERSION/solr-$SOLR_VERSION$SOLR_DIST.tgz"-ARG SOLR_DIST_URL="https://www.apache.org/dist/solr/solr/$SOLR_VERSION/solr-$SOLR_VERSION$SOLR_DIST.tgz"-ARG SOLR_ARCHIVE_URL="https://archive.apache.org/dist/solr/solr/$SOLR_VERSION/solr-$SOLR_VERSION$SOLR_DIST.tgz"+# This server must support downloading at: ${SOLR_DOWNLOAD_SERVER}/${SOLR_VERSION}/solr-${SOLR_VERSION}(-slim).tgz(.asc)+ARG SOLR_DOWNLOAD_SERVER="https://www.apache.org/dyn/closer.lua?action=download&filename=/solr/solr"RUN set -ex; \apt-get update; \apt-get -y --no-install-recommends install wget gpg gnupg dirmngr; \rm -rf /var/lib/apt/lists/*; \- export GNUPGHOME="/tmp/gnupg_home"; \- mkdir -p "$GNUPGHOME"; \- chmod 700 "$GNUPGHOME"; \- echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf"; \- if [ -n "$SOLR_KEYS" ]; then \- # Install all Solr GPG Keys to start- wget -nv "https://downloads.apache.org/solr/KEYS" -O- | \- gpg --batch --import --key-origin 'url,https://downloads.apache.org/solr/KEYS'; \- # Save just the release key- release_keys="$(gpg --batch --export -a ${SOLR_KEYS})"; \- rm -rf "$GNUPGHOME"/*; \- echo "${release_keys}" | gpg --batch --import; \- fi; \+ export SOLR_BINARY="solr-$SOLR_VERSION$SOLR_DIST.tgz"; \MAX_REDIRECTS=3; \- if [ -n "$SOLR_DOWNLOAD_URL" ]; then \- # If a custom URL is defined, we download from non-ASF mirror URL and allow more redirects and skip GPG step- # This takes effect only if the SOLR_DOWNLOAD_URL build-arg is specified, typically in downstream Dockerfiles- MAX_REDIRECTS=4; \- SKIP_GPG_CHECK=true; \- elif [ -n "$SOLR_DOWNLOAD_SERVER" ]; then \- SOLR_DOWNLOAD_URL="$SOLR_DOWNLOAD_SERVER/$SOLR_VERSION/solr-$SOLR_VERSION$SOLR_DIST.tgz"; \- fi; \- for url in $SOLR_DOWNLOAD_URL $SOLR_CLOSER_URL $SOLR_DIST_URL $SOLR_ARCHIVE_URL; do \- if [ -f "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz" ]; then break; fi; \- echo "downloading $url"; \- if wget -t 10 --max-redirect $MAX_REDIRECTS --retry-connrefused -nv "$url" -O "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz"; then break; else rm -f "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz"; fi; \- done; \- if [ ! -f "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz" ]; then echo "failed all download attempts for solr-$SOLR_VERSION$SOLR_DIST.tgz"; exit 1; fi; \+ case "${SOLR_DOWNLOAD_SERVER}" in \+ (*"apache.org"*);; \+ (*) \+ # If a non-ASF URL is provided, allow more redirects and skip GPG step.+ MAX_REDIRECTS=4 && \+ SKIP_GPG_CHECK=true;; \+ esac; \+ export DOWNLOAD_URL="$SOLR_DOWNLOAD_SERVER/$SOLR_VERSION/$SOLR_BINARY"; \+ echo "downloading $DOWNLOAD_URL"; \+ if ! wget -t 10 --max-redirect $MAX_REDIRECTS --retry-connrefused -nv "$DOWNLOAD_URL" -O "/opt/$SOLR_BINARY"; then rm -f "/opt/$SOLR_BINARY"; fi; \+ if [ ! -f "/opt/$SOLR_BINARY" ]; then echo "failed download attempt for $SOLR_BINARY"; exit 1; fi; \+ echo "$SOLR_SHA512 */opt/$SOLR_BINARY" | sha512sum -c -; \if [ -z "$SKIP_GPG_CHECK" ]; then \- echo "downloading $SOLR_ARCHIVE_URL.asc"; \- wget -nv "$SOLR_ARCHIVE_URL.asc" -O "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz.asc"; \- echo "$SOLR_SHA512 */opt/solr-$SOLR_VERSION$SOLR_DIST.tgz" | sha512sum -c -; \- (>&2 ls -l "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz" "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz.asc"); \- gpg --batch --verify "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz.asc" "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz"; \+ # Setup GPG \+ export GNUPGHOME="/tmp/gnupg_home"; \+ mkdir -p "$GNUPGHOME"; \+ chmod 700 "$GNUPGHOME"; \+ echo "disable-ipv6" >> "$GNUPGHOME/dirmngr.conf"; \+ if [ -n "$SOLR_KEYS" ]; then \+ # Install all Solr GPG Keys to start+ wget -nv "https://downloads.apache.org/solr/KEYS" -O- | \+ gpg --batch --import --key-origin 'url,https://downloads.apache.org/solr/KEYS'; \+ # Save just the release key+ release_keys="$(gpg --batch --export -a ${SOLR_KEYS})"; \+ rm -rf "$GNUPGHOME"/*; \+ echo "${release_keys}" | gpg --batch --import; \+ fi; \+ # Do GPG Checks+ echo "downloading $DOWNLOAD_URL.asc"; \+ wget -nv "$DOWNLOAD_URL.asc" -O "/opt/$SOLR_BINARY.asc"; \+ (>&2 ls -l "/opt/$SOLR_BINARY" "/opt/$SOLR_BINARY.asc"); \+ gpg --batch --verify "/opt/$SOLR_BINARY.asc" "/opt/$SOLR_BINARY"; \+ # Cleanup GPG+ { command -v gpgconf; gpgconf --kill all || :; }; \+ rm -r "$GNUPGHOME"; \else \echo "Skipping GPG validation due to non-Apache build"; \fi; \- { command -v gpgconf; gpgconf --kill all || :; }; \- rm -r "$GNUPGHOME"; \- tar -C /opt --extract --preserve-permissions --file "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz"; \- rm "/opt/solr-$SOLR_VERSION$SOLR_DIST.tgz"*; \+ tar -C /opt --extract --preserve-permissions --file "/opt/$SOLR_BINARY"; \+ rm "/opt/$SOLR_BINARY"*; \apt-get -y remove gpg dirmngr && apt-get -y autoremove;diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex e85ab412263..f468fc58830 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -73,15 +73,24 @@ Please refer to the https://solr.apache.org/downloads.html[Solr Downloads] site=== Shard Management* Solr now provides an xref:deployment-guide:shard-management.adoc#installsharddata["Install Shard"] API to allow users who have built (per-shard) indices offline to import them into SolrCloud shards.-== Solr 9.3=== Solr CLI-* the bin/solr -i and bin/solr -info removed in favour of bin/solr status command.+* The `bin/solr -i` and `bin/solr -info` options were removed in favour of the `bin/solr status` command.=== Security* Use of `stream.file`, `stream.url` and `stream.body` params are no longer enabled via configuration in solrconfig.xml, nor dynamic equivalents with the config API.Older configuration now does nothing.Instead, set an env var: SOLR_ENABLE_REMOTE_STREAMING or SOLR_ENABLE_STREAM_BODY or system property equivalents.+=== Official Docker Image+* The customization of the Official Solr Dockerfile has been changed.+The customization options `SOLR_DOWNLOAD_URL`, `SOLR_CLOSER_URL`, `SOLR_DIST_URL` and `SOLR_ARCHIVE_URL`, have been removed.+The only way to specify a custom location for the Solr binaries is through `SOLR_DOWNLOAD_SERVER`.+If the server URL contains `apache.org`, then the Dockerfile will check gpg signature files.+If the server URL does not contain `apache.org`, then the gpg signature checking will be skipped.+++It is still strongly recommended to use the Dockerfile included in the Solr binary TGZs if you want to build Solr images with custom versions of Solr.+The custom version of Solr will include this Dockerfile when it is built.+== Solr 9.2=== Upgrade to Jetty 10.x* Solr upgraded to Jetty 10.x from 9.x due to Jetty 9.x is now end of life. Jetty 10.x has a Java 11 minimum and matches Solr 9 minimum Java version. Jetty logging has been replaced with slf4j again matching Solr. See https://webtide.com/jetty-10-and-11-have-arrived/ for additional Jetty 10.x highlights.