Apache Solr open-source search software
SOLR-15698 No longer check for release status on mirrors (#348)
Jan Høydahl3437977069fa
diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.mdindex 8eb3169b305..f2024b72643 100644--- a/dev-tools/scripts/README.md+++ b/dev-tools/scripts/README.md@@ -144,25 +144,6 @@ and prints a regular expression that will match all of them--no-git Do not run "git" at all--iters N Number of iterations per test suite (default: 5)-### poll-mirrors.py-- usage: poll-mirrors.py [-h] [-version VERSION] [-path PATH]- [-interval INTERVAL] [-details] [-once]-- Periodically checks that all Lucene/Solr mirrors contain either a copy of a- release or a specified path-- optional arguments:- -h, --help show this help message and exit- -version VERSION, -v VERSION- Lucene/Solr version to check- -path PATH, -p PATH instead of a versioned release, check for- some/explicit/path- -interval INTERVAL, -i INTERVAL- seconds to wait before re-querying mirrors- -details, -d print missing mirror URLs- -once, -o run only once-### githubPRs.pyusage: githubPRs.py [-h] [--json] [--token TOKEN]diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.pydeleted file mode 100755index 86fb7ecaddd..00000000000--- a/dev-tools/scripts/poll-mirrors.py+++ /dev/null@@ -1,175 +0,0 @@-#!/usr/bin/env python3-#-# vim: softtabstop=2 shiftwidth=2 expandtab-#-# Python port of poll-mirrors.pl-#-# This script is designed to poll download sites after posting a release-# and print out notice as each becomes available. The RM can use this-# script to delay the release announcement until the release can be-# downloaded.-#-#-# Licensed to the Apache Software Foundation (ASF) under one or more-# contributor license agreements. See the NOTICE file distributed with-# this work for additional information regarding copyright ownership.-# The ASF licenses this file to You under the Apache License, Version 2.0-# (the "License"); you may not use this file except in compliance with-# the License. You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-#-import argparse-import datetime-import ftplib-import re-import sys-import time--from urllib.parse import urlparse-from multiprocessing import Pool-import http.client as http---def p(s):- sys.stdout.write(s)- sys.stdout.flush()---def mirror_contains_file(url):- url = urlparse(url)-- if url.scheme == 'https':- return https_file_exists(url)- elif url.scheme == 'http':- return http_file_exists(url)- elif url.scheme == 'ftp':- return ftp_file_exists(url)---def http_file_exists(url):- exists = False-- try:- conn = http.HTTPConnection(url.netloc)- conn.request('HEAD', url.path)- response = conn.getresponse()-- exists = response.status == 200- except:- pass-- return exists--def https_file_exists(url):- exists = False-- try:- conn = http.HTTPSConnection(url.netloc)- conn.request('HEAD', url.path)- response = conn.getresponse()- exists = response.status == 200- except:- pass-- return exists--def ftp_file_exists(url):- listing = []- try:- conn = ftplib.FTP(url.netloc)- conn.login()- listing = conn.nlst(url.path)- conn.quit()- except Exception as e:- pass-- return len(listing) > 0---def check_mirror(url):- if mirror_contains_file(url):- p('.')- return None- else:- p('\nFAIL: ' + url + '\n' if args.details else 'X')- return url--if __name__ == '__main__':- desc = 'Periodically checks that all Lucene/Solr mirrors contain either a copy of a release or a specified path'- parser = argparse.ArgumentParser(description=desc)- parser.add_argument('-version', '-v', help='Lucene/Solr version to check')- parser.add_argument('-path', '-p', help='instead of a versioned release, check for some/explicit/path')- parser.add_argument('-interval', '-i', help='seconds to wait before re-querying mirrors', type=int, default=300)- parser.add_argument('-details', '-d', help='print missing mirror URLs', action='store_true', default=False)- parser.add_argument('-once', '-o', help='run only once', action='store_true', default=False)- args = parser.parse_args()-- if (args.version is None and args.path is None) \- or (args.version is not None and args.path is not None):- p('You must specify either -version or -path but not both!\n')- sys.exit(1)-- try:- conn = http.HTTPConnection('www.apache.org')- conn.request('GET', '/mirrors/')- response = conn.getresponse()- html = response.read()- except Exception as e:- p('Unable to fetch the Apache mirrors list!\n')- sys.exit(1)-- mirror_path = args.path if args.path is not None else 'lucene/java/{}/changes/Changes.html'.format(args.version)- maven_url = None if args.version is None else 'https://repo1.maven.org/maven2/' \- 'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)- maven_available = False-- pending_mirrors = []- for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):- row = match.group(1)- if not '<TD>ok</TD>' in row:- # skip bad mirrors- continue-- match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)- if match:- pending_mirrors.append(match.group(1) + mirror_path)-- total_mirrors = len(pending_mirrors)-- label = args.version if args.version is not None else args.path- while True:- p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))- p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))- if maven_url is not None and not maven_available:- p(' and Maven Central')- p('...\n')-- if maven_url is not None and not maven_available:- maven_available = mirror_contains_file(maven_url)-- start = time.time()- with Pool(processes=5) as pool:- pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors)))- stop = time.time()- remaining = args.interval - (stop - start)-- available_mirrors = total_mirrors - len(pending_mirrors)-- if maven_url is not None:- p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if maven_available else ' not '))- p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'- .format(label, available_mirrors, total_mirrors, available_mirrors * 100 / total_mirrors))- if len(pending_mirrors) == 0 or args.once == True:- break-- if remaining > 0:- p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))- time.sleep(remaining)-diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.pyindex b10fada4ff5..c05a542b405 100755--- a/dev-tools/scripts/releaseWizard.py+++ b/dev-tools/scripts/releaseWizard.py@@ -1973,6 +1973,25 @@ def prepare_announce_solr(todo):return True+def check_artifacts_available(todo):+ try:+ cdnUrl = expand_jinja("https://dlcdn.apache.org/solr/{{ release_version }}/solr-{{ release_version }}-src.tgz.asc")+ load(cdnUrl)+ print("Found %s" % cdnUrl)+ except Exception as e:+ print("Could not fetch %s (%s)" % (cdnUrl, e))+ return False++ try:+ mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/solr/solr-core/{{ release_version }}/solr-core-{{ release_version }}.jar.asc")+ load(mavenUrl)+ print("Found %s" % mavenUrl)+ except Exception as e:+ print("Could not fetch %s (%s)" % (mavenUrl, e))+ return False++ return True+def set_java_home(version):os.environ['JAVA_HOME'] = state.get_java_home_for_version(version)os.environ['JAVACMD'] = state.get_java_cmd_for_version(version)diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yamlindex 4b6d31bd9fa..91436c2b3ca 100644--- a/dev-tools/scripts/releaseWizard.yaml+++ b/dev-tools/scripts/releaseWizard.yaml@@ -907,8 +907,8 @@ groups:- https://www.apache.org/foundation/voting.html- !TodoGroupid: publish- title: Publishing to the ASF Mirrors- description: Once the vote has passed, the release may be published to the ASF Mirrors and to Maven Central.+ title: Publishing to the ASF Distribution Directory+ description: Once the vote has passed, the release may be published to the ASF Distribution Directory and to Maven Central.todos:- !Todoid: tag_release@@ -937,7 +937,7 @@ groups:commands: !Commandsroot_folder: '{{ git_checkout_folder }}'confirm_each_command: false- commands_text: This will remove maven artifacts so they do not end up in the mirrors+ commands_text: This will remove maven artifacts so they do not end up in the Distribution Directorycommands:- !Commandcmd: svn rm -m "Delete the lucene maven artifacts" {{ dist_stage_url }}/lucene/maven@@ -1010,22 +1010,22 @@ groups:. Wait and keep clicking refresh until the "Release" button becomes available. Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Lucene/Solr {{ release_version }}".- Maven central should show the release after a short while, but you need to- wait 24 hours to give the Apache mirrors a chance to copy the new release.+ Maven central should show the release after a short whilelinks:- https://wiki.apache.org/lucene-java/PublishMavenArtifacts- https://repository.apache.org/index.html- !Todo- id: check_mirroring- title: Check state of mirroring so far- description: Mark this as complete once a good spread is confirmed- commands: !Commands- root_folder: '{{ git_checkout_folder }}'- commands_text: Run this script to check the number and percentage of mirrors (and Maven Central) that have the release- commands:- - !Command- cmd: python3 -u dev-tools/scripts/poll-mirrors.py -version {{ release_version }} -o- live: true+ id: check_distribution_directory+ depends: publish_maven+ title: Check that artifacts are available+ function: check_artifacts_available+ description: |+ The task will attempt to fetch https://dlcdn.apache.org/solr/{{ release_version }}/solr-{{ release_version }}-src.tgz.asc+ to validate ASF repo, and https://repo1.maven.org/maven2/org/apache/solr/solr-core/{{ release_version }}/solr-core-{{ release_version }}.jar.asc+ to validate Maven repo.++ If the check fails, please re-run the task, until it succeeds.+- !TodoGroupid: websitetitle: Update the website@@ -1607,36 +1607,36 @@ groups:- https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions- https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions- !Todo- id: stop_mirroring- title: Stop mirroring old releases+ id: stop_promoting_old+ title: Stop promoting old releasesdescription: |- Shortly after new releases are first mirrored, they are automatically copied to the archives.+ Shortly after new releases are first published, they are automatically copied to the archives.Only the latest point release from each active branch should be kept under the Lucene PMCsvnpubsub areas `dist/releases/lucene/` and `dist/releases/solr/`. Older releases can besafely deleted, since they are already backed up in the archives.- Currenlty these versions are in the mirrors:+ Currenlty these versions exist in the distribution directory:*{{ mirrored_versions|join(', ') }}*The commands below will remove old versions automatically. If this suggestion is wrong,please do *not* execute the commands automatically, but edit the command and run manually.- Versions to be deleted from the mirrors are:+ Versions to be deleted from the distribution directory are:*{{ mirrored_versions_to_delete|join(', ') }}*commands: !Commandsroot_folder: '{{ git_checkout_folder }}'commands_text: |- Run these commands to delete proposed versions from mirrors.+ Run these commands to delete proposed versions from distribution directory.WARNING: Validate that the proposal is correct!commands:- !Commandcmd: |- svn rm -m "Stop mirroring old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}+ svn rm -m "Stop publishing old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}logfile: svn-rm-lucene.log- !Commandcmd: |- svn rm -m "Stop mirroring old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}- logfile: svn-rm-lucene.log+ svn rm -m "Stop publishing old Solr releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/solr/{{ ver }}{% endfor %}+ logfile: svn-rm-solr.log