Apache Lucene open-source search software
build: add missing type-hints to dev-tools and switch to ty
dependabot[bot]409fcc15829d
diff --git a/dev-tools/scripts/Makefile b/dev-tools/scripts/Makefileindex 44fe192a85e..d94559b9323 100644--- a/dev-tools/scripts/Makefile+++ b/dev-tools/scripts/Makefile@@ -20,13 +20,13 @@ PYTHON?=python3VENV=${PWD}/.venv# don't behave strangely if these files exist-.PHONY: lint format reformat autofix ruff ruff-fix pyright env clean smoke-test+.PHONY: lint format reformat autofix ruff ruff-fix ty env clean smoke-test# list of directories we checkSOURCES=$(wildcard *.py)# check formatting, linting, and types-lint: format ruff pyright+lint: format ruff ty# check all formattingformat: env@@ -56,9 +56,9 @@ ruff-fix: env$(VENV)/bin/ruff check --fix $(SOURCES)# checks types-pyright: env- # type-check sources with basedpyright- $(VENV)/bin/basedpyright $(SOURCES)+ty: env+ # type-check sources with ty+ $(VENV)/bin/ty check $(SOURCES)# runs smoketester against a releasesmoke-test: envdiff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.pyindex fe40ed5ebeb..14791407db3 100755--- a/dev-tools/scripts/addBackcompatIndexes.py+++ b/dev-tools/scripts/addBackcompatIndexes.py@@ -34,7 +34,7 @@ import urllib.requestimport scriptutil-def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str):+def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str) -> None:if not current_version.is_back_compat_with(index_version):prefix = "unsupported"else:@@ -79,7 +79,7 @@ def create_and_add_index(source: str, indextype: str, index_version: scriptutil.print("done")-def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version):+def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version) -> None:print(" adding new indexes to backcompat tests...", end="", flush=True)module = "lucene/backward-codecs"@@ -91,12 +91,12 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:strip_dash_suffix_re = re.compile(r"-.*")- def find_version(x: str):+ def find_version(x: str) -> scriptutil.Version:x = x.strip()x = re.sub(strip_dash_suffix_re, "", x) # remove the -suffix if anyreturn scriptutil.Version.parse(x)- def edit(buffer: list[str], _match: re.Match[str], line: str):+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool:v = find_version(line)changed = Falseif v.on_or_after(index_version):@@ -106,7 +106,7 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:buffer.append(line)return changed- def append(buffer: list[str], changed: bool):+ def append(buffer: list[str], changed: bool) -> bool:if changed:return changedif not buffer[len(buffer) - 1].endswith("\n"):@@ -118,13 +118,13 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:print("done" if changed else "uptodate")-def check_backcompat_tests():+def check_backcompat_tests() -> None:print(" checking backcompat tests...", end="", flush=True)scriptutil.run("./gradlew -p lucene/backward-codecs test --tests TestGenerateBwcIndices")print("ok")-def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str):+def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str) -> bool:url = "http://dlcdn.apache.org/lucene/java/%s/%s" % (version, remotename)try:urllib.request.urlretrieve(url, localname)@@ -135,7 +135,7 @@ def download_from_cdn(version: scriptutil.Version, remotename: str, localname: sraise e-def download_from_archives(version: scriptutil.Version, remotename: str, localname: str):+def download_from_archives(version: scriptutil.Version, remotename: str, localname: str) -> bool:url = "http://archive.apache.org/dist/lucene/java/%s/%s" % (version, remotename)try:urllib.request.urlretrieve(url, localname)@@ -146,7 +146,7 @@ def download_from_archives(version: scriptutil.Version, remotename: str, localnaraise e-def download_release(version: scriptutil.Version, temp_dir: str, force: bool):+def download_release(version: scriptutil.Version, temp_dir: str, force: bool) -> str:print(" downloading %s source release..." % version, end="", flush=True)source = os.path.join(temp_dir, "lucene-%s" % version)if os.path.exists(source):@@ -169,7 +169,7 @@ def download_release(version: scriptutil.Version, temp_dir: str, force: bool):return source-def read_config():+def read_config() -> argparse.Namespace:parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,description="""\@@ -186,7 +186,7 @@ http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexesreturn c-def main():+def main() -> None:c = read_config()if not os.path.exists(c.temp_dir):Path(c.temp_dir).mkdir(parents=True)diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.pyindex 997a8a96a0f..cdf1f9ec2f6 100755--- a/dev-tools/scripts/addVersion.py+++ b/dev-tools/scripts/addVersion.py@@ -25,11 +25,11 @@ from configparser import ConfigParser, ExtendedInterpolationfrom scriptutil import Version, find_branch_type, find_current_version, run, update_file-def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]):+def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]) -> None:print(" adding new section to %s..." % filename, end="", flush=True)matcher = re.compile(r"\d+\.\d+\.\d+\s+===")- def edit(buffer: list[str], _match: re.Match[str], line: str):+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:if new_version.dot in line:return Nonematch = new_version.previous_dot_matcher.search(line)@@ -45,14 +45,14 @@ def update_changes(filename: str, new_version: Version, init_changes: str, headeprint("done" if changed else "uptodate")-def add_constant(new_version: Version, deprecate: bool):+def add_constant(new_version: Version, deprecate: bool) -> None:filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"print(" adding constant %s..." % new_version.constant, end="", flush=True)constant_prefix = "public static final Version LUCENE_"matcher = re.compile(constant_prefix)prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep="_")- def ensure_deprecated(buffer: list[str]):+ def ensure_deprecated(buffer: list[str]) -> None:last = buffer[-1]if last.strip() != "@Deprecated":spaces = " " * (len(last) - len(last.lstrip()) - 1)@@ -61,7 +61,7 @@ def add_constant(new_version: Version, deprecate: bool):del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'buffer.append(("{0} * @deprecated ({1}) Use latest\n" + "{0} */\n" + "{0}@Deprecated\n").format(spaces, new_version))- def buffer_constant(buffer: list[str], line: str):+ def buffer_constant(buffer: list[str], line: str) -> None:spaces = " " * (len(line) - len(line.lstrip()))buffer.append(("\n{0}/**\n" + "{0} * Match settings and bugs in Lucene's {1} release.\n").format(spaces, new_version))if deprecate:@@ -76,7 +76,7 @@ def add_constant(new_version: Version, deprecate: bool):class Edit:found = -1- def __call__(self, buffer: list[str], _match: re.Match[str], line: str):+ def __call__(self, buffer: list[str], _match: re.Match[str], line: str) -> bool | None:if new_version.constant in line:return None # constant already exists# outer match is just to find lines declaring version constants@@ -102,11 +102,11 @@ def add_constant(new_version: Version, deprecate: bool):print("done" if changed else "uptodate")-def update_build_version(new_version: Version):+def update_build_version(new_version: Version) -> None:print(" changing baseVersion...", end="", flush=True)filename = "build-options.properties"- def edit(buffer: list[str], _match: re.Match[str], line: str):+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:if new_version.dot in line:return Nonebuffer.append("version.base=" + new_version.dot + "\n")@@ -117,12 +117,12 @@ def update_build_version(new_version: Version):print("done" if changed else "uptodate")-def update_latest_constant(new_version: Version):+def update_latest_constant(new_version: Version) -> None:print(" changing Version.LATEST to %s..." % new_version.constant, end="", flush=True)filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"matcher = re.compile("public static final Version LATEST")- def edit(buffer: list[str], _match: re.Match[str], line: str):+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:if new_version.constant in line:return Nonebuffer.append(line.rpartition("=")[0] + ("= %s;\n" % new_version.constant))@@ -132,17 +132,17 @@ def update_latest_constant(new_version: Version):print("done" if changed else "uptodate")-def onerror(x: Exception):+def onerror(x: Exception) -> None:raise x-def check_lucene_version_tests():+def check_lucene_version_tests() -> None:print(" checking lucene version tests...", end="", flush=True)run("./gradlew -p lucene/core test --tests TestVersion")print("ok")-def read_config(current_version: Version):+def read_config(current_version: Version) -> argparse.Namespace:parser = argparse.ArgumentParser(description="Add a new version to CHANGES, to Version.java and build.gradle files")parser.add_argument("version", type=Version.parse)newconf = parser.parse_args()@@ -156,14 +156,14 @@ def read_config(current_version: Version):# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files-def parse_properties_file(filename: str):+def parse_properties_file(filename: str) -> dict[str, str]:contents = open(filename, encoding="ISO-8859-1").read().replace("%", "%%") # Escape interpolation metacharparser = ConfigParser(interpolation=ExtendedInterpolation()) # Handle ${property-name} interpolationparser.read_string("[DUMMY_SECTION]\n" + contents) # Add required sectionreturn dict(parser.items("DUMMY_SECTION"))-def main():+def main() -> None:if not os.path.exists("build-options.properties"):sys.exit("Tool must be run from the root of a source checkout.")current_version = Version.parse(find_current_version())diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.pyindex e028120eaf5..196c5fb9cb8 100755--- a/dev-tools/scripts/buildAndPushRelease.py+++ b/dev-tools/scripts/buildAndPushRelease.py@@ -34,21 +34,21 @@ LOG = "/tmp/release.log"dev_mode = False-def log(msg: str):+def log(msg: str) -> None:f = open(LOG, mode="ab")f.write(msg.encode("utf-8"))f.close()-def run(command: str):+def run(command: str) -> None:log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))- if os.system("%s >> %s 2>&1" % (command, LOG)):+ if os.system("%s >> %s 2>&1" % (command, LOG)): # ty:ignore[deprecated]msg = " FAILED: %s [see log %s]" % (command, LOG)print(msg)raise RuntimeError(msg)-def runAndSendGPGPassword(command: str, password: str):+def runAndSendGPGPassword(command: str, password: str) -> None:p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)f = open(LOG, "ab")while True:@@ -76,7 +76,7 @@ def runAndSendGPGPassword(command: str, password: str):raise RuntimeError(msg)-def load(urlString: str, encoding: str = "utf-8"):+def load(urlString: str, encoding: str = "utf-8") -> str:try:content = urllib.request.urlopen(urlString).read().decode(encoding)except Exception as e:@@ -85,9 +85,9 @@ def load(urlString: str, encoding: str = "utf-8"):return content-def getGitRev():+def getGitRev() -> str:if not dev_mode:- status = os.popen("git status").read().strip()+ status = os.popen("git status").read().strip() # ty:ignore[deprecated]if "nothing to commit, working directory clean" not in status and "nothing to commit, working tree clean" not in status:raise RuntimeError("git clone is dirty:\n\n%s" % status)if "Your branch is ahead of" in status:@@ -95,10 +95,10 @@ def getGitRev():print(" git clone is clean")else:print(" Ignoring dirty git clone due to dev-mode")- return os.popen("git rev-parse HEAD").read().strip()+ return os.popen("git rev-parse HEAD").read().strip() # ty:ignore[deprecated]-def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False):+def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False) -> str:print()print("Prepare release...")if os.path.exists(LOG):@@ -163,7 +163,7 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<", re.IGNORECreDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?", re.IGNORECASE)-def checkDOAPfiles(version: str):+def checkDOAPfiles(version: str) -> None:# In Lucene DOAP file, verify presence of all releases less than the one being produced.errorMessages: list[str] = []for product in ["lucene"]:@@ -196,7 +196,7 @@ def checkDOAPfiles(version: str):raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))-def normalizeVersion(tup: tuple[str, ...]):+def normalizeVersion(tup: tuple[str, ...]) -> str:suffix = ""if tup[-1] is not None and tup[-1].lower() == "-alpha":tup = tup[: (len(tup) - 1)]@@ -211,7 +211,7 @@ def normalizeVersion(tup: tuple[str, ...]):return ".".join(tup) + suffix-def pushLocal(version: str, root: str, rcNum: int, localDir: str):+def pushLocal(version: str, root: str, rcNum: int, localDir: str) -> str:print("Push local [%s]..." % localDir)Path(localDir).mkdir(parents=True)@@ -240,11 +240,11 @@ def pushLocal(version: str, root: str, rcNum: int, localDir: str):return "file://%s/%s" % (Path(localDir).resolve(), dir)-def read_version(_path: str):+def read_version(_path: str) -> str:return scriptutil.find_current_version()-def parse_config():+def parse_config() -> argparse.Namespace:epilogue = textwrap.dedent("""Example usage for a Release Manager:python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1@@ -300,7 +300,7 @@ def parse_config():cwd = os.getcwd()os.chdir(config.root)config.root = os.getcwd() # Absolutize root dir- if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:+ if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2: # ty:ignore[deprecated]parser.error('Root path "%s" is not a valid lucene checkout' % config.root)os.chdir(cwd)global LOG@@ -314,12 +314,12 @@ def parse_config():return config-def check_cmdline_tools(): # Fail fast if there are cmdline tool problems- if os.system("git --version >/dev/null 2>/dev/null"):+def check_cmdline_tools() -> None: # Fail fast if there are cmdline tool problems+ if os.system("git --version >/dev/null 2>/dev/null"): # ty:ignore[deprecated]raise RuntimeError('"git --version" returned a non-zero exit code.')-def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):+def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None) -> None:if gpgKeyID is not None:print(" Verify your gpg key is in the main KEYS file")if local_keys is not None:@@ -364,7 +364,7 @@ def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):exit(2)-def resolve_gpghome():+def resolve_gpghome() -> str | None:for p in [# Linux, macosos.path.join(os.path.expanduser("~"), ".gnupg"),@@ -377,7 +377,7 @@ def resolve_gpghome():return None-def main():+def main() -> None:check_cmdline_tools()c = parse_config()diff --git a/dev-tools/scripts/create_line_file_docs.py b/dev-tools/scripts/create_line_file_docs.pyindex 856ee9e10ee..f8be5df594f 100644--- a/dev-tools/scripts/create_line_file_docs.py+++ b/dev-tools/scripts/create_line_file_docs.py@@ -31,7 +31,7 @@ DEBUG = FalseTARGET_DOC_CHARS = 1024-def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int):+def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int) -> None:bytes_per_chunk = os.path.getsize(file_name_in) / num_seek_pointsseek_points: list[int] = []@@ -82,7 +82,7 @@ re_next_non_word_character = re.compile(r"\W", re.UNICODE)EUROPARL_V7_URL = "https://www.statmt.org/europarl/v7/europarl.tgz"-def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str):+def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str) -> int:"""Splits docs into smallish (~1 KB) sized docs, repeating same title and date"""doc_count = 0while len(body_string) > 0:@@ -107,7 +107,7 @@ def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, bodyreturn doc_count-def sample_europarl():+def sample_europarl() -> None:# download europarl.tgz v7, if not already here (in cwd):file_name = "europarl.tgz"if not os.path.exists(file_name):diff --git a/dev-tools/scripts/diff_lucene_changes.py b/dev-tools/scripts/diff_lucene_changes.pyindex 4d8e5121f39..df65ca97288 100644--- a/dev-tools/scripts/diff_lucene_changes.py+++ b/dev-tools/scripts/diff_lucene_changes.py@@ -31,7 +31,7 @@ to users and avoid future annoying conflicts on backport.# e.g. python3 -u diff_lucene_changes.py branch_9_9 main 9.9.0-def get_changes_url(branch_name: str):+def get_changes_url(branch_name: str) -> str:if os.path.isdir(branch_name):url = f"file://{branch_name}/lucene/CHANGES.txt"else:@@ -40,13 +40,13 @@ def get_changes_url(branch_name: str):return url-def extract_release_section(changes_txt: bytes, release_name: str):+def extract_release_section(changes_txt: bytes, release_name: str) -> bytes:match = re.search(f"=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$", changes_txt.decode("utf-8"), re.MULTILINE | re.DOTALL)assert matchreturn match.group(1).encode("utf-8")-def main():+def main() -> None:if len(sys.argv) < 3 or len(sys.argv) > 5:print("\nUsage: python3 -u dev-tools/scripts/diff_lucene_changes.py <branch1-or-local-clone> <branch2-or-local-clone> <release-name> [diff-commandline-extras]\n")print(' e.g.: python3 -u dev-tools/scripts/diff_lucene_changes.py branch_9_9 /l/trunk 9.9.0 "-w"\n')diff --git a/dev-tools/scripts/githubPRs.py b/dev-tools/scripts/githubPRs.pyindex b6d425c17bf..efa93d5cfcd 100755--- a/dev-tools/scripts/githubPRs.py+++ b/dev-tools/scripts/githubPRs.py@@ -20,6 +20,7 @@ issue number in title, and the ones where the linked JIRA is already closedimport osimport sys+from argparse import Namespacesys.path.append(os.path.dirname(__file__))import argparse@@ -35,7 +36,7 @@ if TYPE_CHECKING:from github.PullRequest import PullRequest-def read_config():+def read_config() -> Namespace:parser = argparse.ArgumentParser(description="Find open Pull Requests that need attention")parser.add_argument("--json", action="store_true", default=False, help="Output as json")parser.add_argument("--html", action="store_true", default=False, help="Output as html")@@ -44,14 +45,12 @@ def read_config():return newconf-def out(text: str):- global conf+def out(conf: Namespace, text: str) -> None:if not (conf.json or conf.html):print(text)-def make_html(dict: dict[Any, Any]):- global conf+def make_html(dict: dict[Any, Any]) -> str:template = Environment(loader=BaseLoader()).from_string("""<h1>Lucene Github PR report</h1>@@ -74,8 +73,7 @@ def make_html(dict: dict[Any, Any]):return template.render(dict)-def main():- global conf+def main() -> None:conf = read_config()token = conf.token if conf.token is not None else Noneif token:@@ -86,9 +84,9 @@ def main():result: dict[str, Any] = {}repo = gh.get_repo("apache/lucene")open_prs = repo.get_pulls(state="open")- out("Lucene Github PR report")- out("============================")- out("Number of open Pull Requests: %s" % open_prs.totalCount)+ out(conf, "Lucene Github PR report")+ out(conf, "============================")+ out(conf, "Number of open Pull Requests: %s" % open_prs.totalCount)result["open_count"] = open_prs.totalCountlack_jira = list(filter(lambda x: not re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))@@ -97,11 +95,11 @@ def main():for pr in lack_jira:lack_jira_list.append({"title": pr.title, "number": pr.number, "user": pr.user.login, "created": pr.created_at.strftime("%Y-%m-%d")})result["no_jira"] = lack_jira_list- out("\nPRs lacking JIRA reference in title")+ out(conf, "\nPRs lacking JIRA reference in title")for pr in lack_jira_list:- out(" #%s: %s %s (%s)" % (pr["number"], pr["created"], pr["title"], pr["user"]))+ out(conf, " #%s: %s %s (%s)" % (pr["number"], pr["created"], pr["title"], pr["user"]))- out("\nOpen PRs with a resolved JIRA")+ out(conf, "\nOpen PRs with a resolved JIRA")has_jira = list(filter(lambda x: re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))issue_ids: list[str] = []@@ -136,7 +134,7 @@ def main():closed_jiras.sort(key=lambda r: r["pr_number"], reverse=True)for issue in closed_jiras:- out(" #%s: %s %s %s: %s (%s)" % (issue["pr_number"], issue["status"], issue["resolution_date"], issue["issue_key"], issue["issue_summary"], issue["assignee"]))+ out(conf, " #%s: %s %s %s: %s (%s)" % (issue["pr_number"], issue["status"], issue["resolution_date"], issue["issue_key"], issue["issue_summary"], issue["assignee"]))result["closed_jira_count"] = len(resolved_jiras)result["closed_jira"] = closed_jirasdiff --git a/dev-tools/scripts/pyproject.toml b/dev-tools/scripts/pyproject.tomlindex c19b333c397..ab69b478502 100644--- a/dev-tools/scripts/pyproject.toml+++ b/dev-tools/scripts/pyproject.toml@@ -1,26 +1,16 @@-[tool.pyright]-pythonVersion = "3.12"-venvPath = "."-venv = ".venv"-typeCheckingMode = "strict"-reportCallInDefaultInitializer = "error"-reportImplicitStringConcatenation = "error"-reportImportCycles = "error"-reportImplicitOverride = "error"-reportPropertyTypeMismatch = "error"-reportUninitializedInstanceVariable = "error"-reportUnnecessaryTypeIgnoreComment = "error"+[tool.ty.environment]+python-version = "3.12"-# This is a big TODO list of current typing problems-# Pick an item off the list, change to "error", and send a pull request to fix it!-# You can find more information on a rule at https://github.com/microsoft/pyright/blob/main/docs/configuration.md-reportConstantRedefinition = "hint" # Requires fixing nonstandard variable conventions-reportMissingSuperCall = "hint" # Requires adding missing super().__init__() calls-reportMissingTypeStubs = "hint" # Can be fixed by switching 'ics' to alternative such as 'icalendar'-reportPrivateUsage = "hint" # Can be fixed by using TLS properly (Ruff S323)-reportUnnecessaryComparison = "hint" # Requires cleaning up some dead code-reportUntypedNamedTuple = "hint" # Requires moving to different type of named tuple-reportUnusedCallResult = "none" # Forces you to assign unused retvals to '_', very noisy.+[tool.ty.rules]+division-by-zero = "warn"+missing-override-decorator = "warn"+possibly-missing-attribute = "warn"+possibly-missing-import = "warn"+possibly-unresolved-reference = "warn"+unsupported-dynamic-base = "warn"++[tool.ty.terminal]+error-on-warning = true[tool.ruff]target-version = "py312"@@ -37,11 +27,6 @@ ignore = [# You can find more information on a rule at https://docs.astral.sh/ruff/rules/"A001", # variable name is shadowing a python builtin"A002", # function argument name is shadowing a python builtin- "ANN001", # missing type annotation for function argument- "ANN201", # missing return annotation for public function- "ANN202", # missing return annotation for private function- "ANN204", # missing return annotation for special method- "ANN206", # missing return annotation for classmethod"ANN401", # use of Any as function parameter"ARG005", # unused lambda argument"B006", # do not use mutable datastructures for arg defaultsdiff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.pyindex 6fc70bbae7e..37de9a936aa 100755--- a/dev-tools/scripts/releaseWizard.py+++ b/dev-tools/scripts/releaseWizard.py@@ -67,7 +67,7 @@ editor = None# Edit this to add other global jinja2 variables or filters-def expand_jinja(text: str, vars: dict[str, Any] | None = None):+def expand_jinja(text: str, vars: dict[str, Any] | None = None) -> str:global_vars: dict[str, Any] = OrderedDict({"script_version": state.script_version,@@ -135,7 +135,7 @@ def expand_jinja(text: str, vars: dict[str, Any] | None = None):return filled-def replace_templates(text: str):+def replace_templates(text: str) -> str:tpl_lines: list[str] = []for line in text.splitlines():if match := re.search(r"^\(\( template=(.+?) \)\)", line):@@ -146,11 +146,11 @@ def replace_templates(text: str):return "\n".join(tpl_lines)-def getScriptVersion():+def getScriptVersion() -> str:return scriptutil.find_current_version()-def get_editor():+def get_editor() -> str:global editorif editor is None:if "EDITOR" in os.environ:@@ -166,7 +166,7 @@ def get_editor():return editor-def check_prerequisites(todo: Any = None):+def check_prerequisites(todo: "Todo | None" = None) -> bool:if sys.version_info < (3, 4):sys.exit("Script requires Python v3.4 or later")try:@@ -201,11 +201,11 @@ def check_prerequisites(todo: Any = None):epoch = datetime.fromtimestamp(timestamp=0, tz=UTC)-def unix_time_millis(dt: datetime):+def unix_time_millis(dt: datetime) -> int:return int((dt - epoch).total_seconds() * 1000.0)-def bootstrap_todos(todo_list: list[Any]):+def bootstrap_todos(todo_list: list[Any]) -> list[Any]:# Establish links from commands to to_do for finding todo varsfor tg in todo_list:if dry_run:@@ -227,7 +227,7 @@ def bootstrap_todos(todo_list: list[Any]):return todo_list-def maybe_remove_rc_from_svn():+def maybe_remove_rc_from_svn() -> None:todo = state.get_todo_by_id("import_svn")if todo and todo.is_done():print("import_svn done")@@ -253,7 +253,7 @@ class SecretYamlObject(yaml.YAMLObject):@classmethod@override- def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any):+ def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:print("Dumping object %s" % type(data))new_data = copy.deepcopy(data)@@ -263,17 +263,17 @@ class SecretYamlObject(yaml.YAMLObject):for item in data.__dict__:if item in new_data.__dict__ and new_data.__dict__[item] is None:del new_data.__dict__[item]- return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style) # TODO: fix me # pyright: ignore[reportUnknownMemberType]+ return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style)-def str_presenter(dumper: yaml.Dumper, data: str):+def str_presenter(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode:if len(data.split("\n")) > 1: # check for multiline string- return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") # TODO: fix me # pyright: ignore[reportUnknownMemberType]- return dumper.represent_scalar("tag:yaml.org,2002:str", data) # TODO: fix me # pyright: ignore[reportUnknownMemberType]+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")+ return dumper.represent_scalar("tag:yaml.org,2002:str", data)class ReleaseState:- def __init__(self, config_path: str, release_version: str, script_version: str):+ def __init__(self, config_path: str, release_version: str, script_version: str) -> None:self.script_version = script_versionself.config_path = config_pathself.todo_groups: list[TodoGroup] = []@@ -291,7 +291,7 @@ class ReleaseState:self.script_branch_type = "feature"self.set_release_version(release_version)- def set_release_version(self, version: str):+ def set_release_version(self, version: str) -> None:self.validate_release_version(self.script_branch_type, self.script_branch, version)self.release_version = versionv = Version.parse(version)@@ -306,32 +306,32 @@ class ReleaseState:else:self.release_type = "bugfix"- def is_released(self):+ def is_released(self) -> bool:todo = self.get_todo_by_id("announce_lucene")assert todoreturn todo.is_done()- def get_gpg_key(self):+ def get_gpg_key(self) -> str | None:gpg_task = self.get_todo_by_id("gpg")assert gpg_taskif gpg_task.is_done():return gpg_task.get_state()["gpg_key"]return None- def get_release_date(self):+ def get_release_date(self) -> datetime | None:publish_task = self.get_todo_by_id("publish_maven")assert publish_taskif publish_task.is_done():return unix_to_datetime(publish_task.get_state()["done_date"])return None- def get_release_date_iso(self):+ def get_release_date_iso(self) -> str:release_date = self.get_release_date()if release_date is None:return "yyyy-mm-dd"return release_date.isoformat()[:10]- def get_latest_version(self):+ def get_latest_version(self) -> str:if self.latest_version is None:versions = self.get_mirrored_versions()latest = versions[0]@@ -343,14 +343,14 @@ class ReleaseState:assert state.latest_versionreturn state.latest_version- def get_mirrored_versions(self):+ def get_mirrored_versions(self) -> list[str]:if state.mirrored_versions is None:releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")releases: dict[str, str] = json.loads(releases_str)["lucene"]state.mirrored_versions = [r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith("lucene-"), list(releases.keys()))))]return state.mirrored_versions- def get_mirrored_versions_to_delete(self):+ def get_mirrored_versions_to_delete(self) -> list[str]:versions = self.get_mirrored_versions()to_keep = versionsif state.release_type == "major":@@ -366,11 +366,11 @@ class ReleaseState:raise Exception("Release version %s must have same major version as current minor or lts release")return [ver for ver in versions if ver not in to_keep]- def get_main_version(self):+ def get_main_version(self) -> str:v = Version.parse(self.get_latest_version())return "%s.%s.%s" % (v.major + 1, 0, 0)- def get_latest_lts_version(self):+ def get_latest_lts_version(self) -> str:versions = self.get_mirrored_versions()latest = self.get_latest_version()lts_prefix = "%s." % (Version.parse(latest).major - 1)@@ -381,7 +381,7 @@ class ReleaseState:latest_lts = verreturn latest_lts- def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str):+ def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str) -> None:ver = Version.parse(release_version)# print("release_version=%s, ver=%s" % (release_version, ver))if branch_type == BranchType.release:@@ -402,7 +402,7 @@ class ReleaseState:if not getScriptVersion() == release_version:print("WARNING: Expected release version %s when on branch %s, but got %s" % (getScriptVersion(), branch, release_version))- def get_base_branch_name(self):+ def get_base_branch_name(self) -> str:v = Version.parse(self.release_version)if v.is_major_release():return "main"@@ -412,7 +412,7 @@ class ReleaseState:return self.get_minor_branch_name()return self.release_branch- def clear_rc(self):+ def clear_rc(self) -> None:if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):maybe_remove_rc_from_svn()for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):@@ -426,7 +426,7 @@ class ReleaseState:print("WARN: Failed to clear %s, please do it manually with higher privileges" % self.get_rc_folder())self.save()- def new_rc(self):+ def new_rc(self) -> None:if ask_yes_no("Are you sure? This will abort current RC"):maybe_remove_rc_from_svn()dict = {}@@ -439,7 +439,7 @@ class ReleaseState:self.rc_number += 1self.save()- def to_dict(self):+ def to_dict(self) -> dict[str, Any]:tmp_todos = {}for todo_id in self.todos:t = self.todos[todo_id]@@ -457,7 +457,7 @@ class ReleaseState:dictionary["latest_version"] = self.latest_versionreturn dictionary- def restore_from_dict(self, dictionary: dict[str, Any]):+ def restore_from_dict(self, dictionary: dict[str, Any]) -> None:self.script_version = dictionary["script_version"]assert dictionary["release_version"] == self.release_versionif "start_date" in dictionary:@@ -474,7 +474,7 @@ class ReleaseState:else:print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)- def load(self):+ def load(self) -> None:if os.path.exists(os.path.join(self.config_path, self.release_version, "state.yaml")):state_file = os.path.join(self.config_path, self.release_version, "state.yaml")with open(state_file) as fp:@@ -485,7 +485,7 @@ class ReleaseState:except Exception as e:print("Failed to load state from %s: %s" % (state_file, e))- def save(self):+ def save(self) -> None:print("Saving")if not os.path.exists(os.path.join(self.config_path, self.release_version)):print("Creating folder %s" % os.path.join(self.config_path, self.release_version))@@ -494,7 +494,7 @@ class ReleaseState:with open(os.path.join(self.config_path, self.release_version, "state.yaml"), "w") as fp:yaml.dump(self.to_dict(), fp, sort_keys=False, default_flow_style=False)- def clear(self):+ def clear(self) -> None:self.previous_rcs = {}self.rc_number = 1for t_id in self.todos:@@ -502,72 +502,72 @@ class ReleaseState:t.state = {}self.save()- def get_rc_number(self):+ def get_rc_number(self) -> int:return self.rc_number- def get_current_git_rev(self):+ def get_current_git_rev(self) -> str:try:return run("git rev-parse HEAD", cwd=self.get_git_checkout_folder()).strip()except Exception:return "<git-rev>"- def get_group_by_id(self, id: str):+ def get_group_by_id(self, id: str) -> "TodoGroup | None":lst: list[TodoGroup] = list(filter(lambda x: x.id == id, self.todo_groups))if len(lst) == 1:return lst[0]return None- def get_todo_by_id(self, id: str):+ def get_todo_by_id(self, id: str) -> "Todo | None":lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))if len(lst) == 1:return lst[0]return None- def get_todo_state_by_id(self, id: str):+ def get_todo_state_by_id(self, id: str) -> dict[Any, Any]:lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))if len(lst) == 1:return lst[0].state- return cast("dict[Any, Any]", {})+ return {}- def get_release_folder(self):+ def get_release_folder(self) -> str:folder = os.path.join(self.config_path, self.release_version)if not os.path.exists(folder):print("Creating folder %s" % folder)Path(folder).mkdir(parents=True)return folder- def get_rc_folder(self):+ def get_rc_folder(self) -> str:folder = os.path.join(self.get_release_folder(), "RC%d" % self.rc_number)if not os.path.exists(folder):print("Creating folder %s" % folder)Path(folder).mkdir(parents=True)return folder- def get_dist_folder(self):+ def get_dist_folder(self) -> str:folder = os.path.join(self.get_rc_folder(), "dist")return folder- def get_git_checkout_folder(self):+ def get_git_checkout_folder(self) -> str:folder = os.path.join(self.get_release_folder(), "lucene")return folder- def get_website_git_folder(self):+ def get_website_git_folder(self) -> str:folder = os.path.join(self.get_release_folder(), "lucene-site")return folder- def get_minor_branch_name(self):+ def get_minor_branch_name(self) -> str:latest = state.get_latest_version()v = Version.parse(latest)return "branch_%s_%s" % (v.major, v.minor)- def get_stable_branch_name(self):+ def get_stable_branch_name(self) -> str:if self.release_type == "major":v = Version.parse(self.get_main_version())else:v = Version.parse(self.get_latest_version())return "branch_%sx" % v.major- def get_next_version(self):+ def get_next_version(self) -> str | None:if self.release_type == "major":return "%s.0.0" % (self.release_version_major + 1)if self.release_type == "minor":@@ -576,10 +576,10 @@ class ReleaseState:return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)return None- def get_java_home(self):+ def get_java_home(self) -> str:return self.get_java_home_for_version(self.release_version)- def get_java_home_for_version(self, version: str):+ def get_java_home_for_version(self, version: str) -> str:v = Version.parse(version)java_ver = java_versions[v.major]java_home_var = "JAVA%s_HOME" % java_ver@@ -587,21 +587,21 @@ class ReleaseState:return os.environ[java_home_var]raise Exception("Script needs environment variable %s" % java_home_var)- def get_java_cmd_for_version(self, version: str):+ def get_java_cmd_for_version(self, version: str) -> str:return os.path.join(self.get_java_home_for_version(version), "bin", "java")- def get_java_cmd(self):+ def get_java_cmd(self) -> str:return os.path.join(self.get_java_home(), "bin", "java")- def get_todo_states(self):- states: dict[str, dict[Any, Any]] = {}+ def get_todo_states(self) -> dict[str, dict[Any, Any]]:+ states = {}if self.todos:for todo_id in self.todos:t = self.todos[todo_id]states[todo_id] = copy.deepcopy(t.state)return states- def init_todos(self, groups: list[Any]):+ def init_todos(self, groups: list[Any]) -> None:self.todo_groups = groupsself.todos = {}for g in self.todo_groups:@@ -613,7 +613,7 @@ class TodoGroup(SecretYamlObject):yaml_tag = "!TodoGroup"hidden_fields = []- def __init__(self, id: str, title: str, description: str, todos: list[Any], is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None):+ def __init__(self, id: str, title: str, description: str, todos: "list[Todo]", is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None) -> None:self.id = idself.title = titleself.description = description@@ -623,30 +623,30 @@ class TodoGroup(SecretYamlObject):@classmethod@override- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "TodoGroup":fields = loader.construct_mapping(node, deep=True)return TodoGroup(**cast("dict[str, Any]", fields))- def num_done(self):+ def num_done(self) -> int:return sum(1 for x in self.todos if x.is_done() > 0)- def num_applies(self):+ def num_applies(self) -> int:count = sum(1 for x in self.todos if x.applies(state.release_type))# print("num_applies=%s" % count)return count- def is_done(self):+ def is_done(self) -> bool:# print("Done=%s, applies=%s" % (self.num_done(), self.num_applies()))return self.num_done() >= self.num_applies()- def get_title(self):+ def get_title(self) -> str:# print("get_title: %s" % self.is_done())prefix = ""if self.is_done():prefix = "✓ "return "%s%s (%d/%d)" % (prefix, self.title, self.num_done(), self.num_applies())- def get_submenu(self):+ def get_submenu(self) -> ConsoleMenu:menu = ConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(), clear_screen=False)menu.exit_item = CustomExitItem("Return")for todo in self.get_todos():@@ -654,17 +654,17 @@ class TodoGroup(SecretYamlObject):menu.append_item(todo.get_menu_item())return menu- def get_menu_item(self):+ def get_menu_item(self) -> SubmenuItem:item = SubmenuItem(self.get_title, self.get_submenu())return item- def get_todos(self):+ def get_todos(self) -> "list[Todo]":return self.todos- def in_rc_loop(self):+ def in_rc_loop(self) -> bool:return self.is_in_rc_loop is True- def get_description(self):+ def get_description(self) -> str | None:desc = self.descriptionif desc:return expand_jinja(desc)@@ -696,14 +696,14 @@ class Todo(SecretYamlObject):done: bool | None = None,types: list[str] | str | None = None,links: list[str] | None = None,- commands: Any = None,- user_input: Any = None,+ commands: "Commands | None" = None,+ user_input: "UserInput | None" = None,depends: str | list[str] | None = None,vars: dict[str, Any] | None = None,asciidoc: str | None = None,persist_vars: dict[str, str] | None = None,function: str | None = None,- ):+ ) -> None:self.id = idself.title = titleself.description = description@@ -732,11 +732,11 @@ class Todo(SecretYamlObject):@classmethod@override- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Todo":fields = loader.construct_mapping(node, deep=True)return Todo(**cast("dict[str, Any]", fields))- def get_vars(self):+ def get_vars(self) -> dict[str, str]:myvars: dict[str, str] = {}if self.vars:for k in self.vars:@@ -747,7 +747,7 @@ class Todo(SecretYamlObject):myvars[k] = expand_jinja(val, vars=myvars)return myvars- def set_done(self, is_done: bool | None):+ def set_done(self, is_done: bool | None) -> None:if is_done:self.state["done_date"] = unix_time_millis(datetime.now(tz=UTC))if self.persist_vars:@@ -757,21 +757,21 @@ class Todo(SecretYamlObject):self.state.clear()self.state["done"] = is_done- def applies(self, type: str):+ def applies(self, type: str) -> bool:if self.types:return type in self.typesreturn True- def is_done(self):+ def is_done(self) -> bool:return "done" in self.state and self.state["done"] is True- def get_title(self):+ def get_title(self) -> str:prefix = ""if self.is_done():prefix = "✓ "return expand_jinja("%s%s" % (prefix, self.title), self.get_vars_and_state())- def display_and_confirm(self):+ def display_and_confirm(self) -> None:try:if self.depends:for dep in ensure_list(self.depends):@@ -827,48 +827,48 @@ class Todo(SecretYamlObject):except Exception as e:print("ERROR while executing todo %s (%s)" % (self.get_title(), e))- def get_menu_item(self):+ def get_menu_item(self) -> FunctionItem:return FunctionItem(self.get_title(), self.display_and_confirm)- def clone(self):+ def clone(self) -> "Todo":clone = Todo(self.id, self.title, description=self.description)clone.state = copy.deepcopy(self.state)return clone- def clear(self):+ def clear(self) -> None:self.state.clear()self.set_done(False)- def get_state(self):+ def get_state(self) -> dict[Any, Any]:return self.state- def get_description(self):+ def get_description(self) -> str | None:desc = self.descriptionif desc:return expand_jinja(desc, vars=self.get_vars_and_state())return None- def get_post_description(self):+ def get_post_description(self) -> str | None:if self.post_description:return expand_jinja(self.post_description, vars=self.get_vars_and_state())return None- def get_commands(self):+ def get_commands(self) -> "Commands | None":cmds = self.commandsreturn cmds- def get_asciidoc(self):+ def get_asciidoc(self) -> str | None:if self.asciidoc:return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())return None- def get_vars_and_state(self):+ def get_vars_and_state(self) -> dict[str, str]:d = self.get_vars().copy()d.update(self.get_state())return d-def get_release_version():+def get_release_version() -> str:v = str(input("Which version are you releasing? (x.y.z) "))try:version = Version.parse(v)@@ -879,30 +879,33 @@ def get_release_version():return str(version)-def get_subtitle():+def get_subtitle() -> str:applying_groups = list(filter(lambda x: x.num_applies() > 0, state.todo_groups))done_groups = sum(1 for x in applying_groups if x.is_done())return "Please complete the below checklist (Complete: %s/%s)" % (done_groups, len(applying_groups))-def get_todo_menuitem_title():+def get_todo_menuitem_title() -> str:return "Go to checklist (RC%d)" % (state.rc_number)-def get_releasing_text():+def get_releasing_text() -> str:return "Releasing Lucene %s RC%d" % (state.release_version, state.rc_number)-def get_start_new_rc_menu_title():+def get_start_new_rc_menu_title() -> str:return "Abort RC%d and start a new RC%d" % (state.rc_number, state.rc_number + 1)-def start_new_rc():+def start_new_rc() -> None:state.new_rc()print("Started RC%d" % state.rc_number)-def reset_state():+state: ReleaseState+++def reset_state() -> None:global stateif ask_yes_no("Are you sure? This will erase all current progress"):maybe_remove_rc_from_svn()@@ -910,16 +913,16 @@ def reset_state():state.clear()-def template(name: str, vars: dict[str, Any] | None = None):+def template(name: str, vars: dict[str, Any] | None = None) -> str:return expand_jinja(templates[name], vars=vars)-def help():+def help() -> None:print(template("help"))pause()-def ensure_list(o: Any):+def ensure_list(o: Any) -> list[Any]:if o is None:return cast("list[Any]", [])if not isinstance(o, list):@@ -927,7 +930,7 @@ def ensure_list(o: Any):return cast("list[Any]", o)-def open_file(filename: str):+def open_file(filename: str) -> None:print("Opening file %s" % filename)if platform.system().startswith("Win"):run("start %s" % filename)@@ -935,15 +938,15 @@ def open_file(filename: str):run("open %s" % filename)-def expand_multiline(cmd_txt: str, indent: int = 0):+def expand_multiline(cmd_txt: str, indent: int = 0) -> str:return re.sub(r" +", " %s\n %s" % (Commands.cmd_continuation_char, " " * indent), cmd_txt)-def unix_to_datetime(unix_stamp: int):+def unix_to_datetime(unix_stamp: int) -> datetime:return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=UTC)-def generate_asciidoc():+def generate_asciidoc() -> None:base_filename = os.path.join(state.get_release_folder(), "lucene_release_%s" % (state.release_version.replace(r"\.", "_")))filename_adoc = "%s.adoc" % base_filename@@ -1025,7 +1028,7 @@ def generate_asciidoc():pause()-def load_rc():+def load_rc() -> dict[str, Any]:lucenerc = os.path.expanduser("~/.lucenerc")try:with open(lucenerc) as fp:@@ -1034,7 +1037,7 @@ def load_rc():return cast("dict[str, Any]", {})-def store_rc(release_root: str, release_version: str | None = None):+def store_rc(release_root: str, release_version: str | None = None) -> None:lucenerc = os.path.expanduser("~/.lucenerc")dict = {}dict["root"] = release_root@@ -1044,7 +1047,7 @@ def store_rc(release_root: str, release_version: str | None = None):json.dump(dict, fp, indent=2)-def release_other_version():+def release_other_version() -> None:if not state.is_released():maybe_remove_rc_from_svn()store_rc(state.config_path, None)@@ -1052,28 +1055,28 @@ def release_other_version():sys.exit(0)-def file_to_string(filename: str):+def file_to_string(filename: str) -> str:with open(filename, encoding="utf8") as f:return f.read().strip()-def download_keys():+def download_keys() -> None:download("KEYS", "https://archive.apache.org/dist/lucene/KEYS", state.config_path)-def keys_downloaded():+def keys_downloaded() -> bool:return os.path.exists(os.path.join(state.config_path, "KEYS"))-def dump_yaml():+def dump_yaml() -> None:file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")yaml.add_representer(str, str_presenter)- yaml.Dumper.ignore_aliases = lambda self, data: True # TODO: fix me # pyright: ignore[reportUnknownLambdaType]+ yaml.Dumper.ignore_aliases = lambda self, data: True # ty:ignore[invalid-assignment]dump_obj: dict[str, Any] = {"templates": templates, "groups": state.todo_groups}yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)-def parse_config():+def parse_config() -> argparse.Namespace:description = "Script to guide a RM through the whole release process"parser = argparse.ArgumentParser(description=description, epilog="Go push that release!", formatter_class=argparse.RawDescriptionHelpFormatter)parser.add_argument("--dry-run", dest="dry", action="store_true", default=False, help="Do not execute any commands, but echo them instead. Display extra debug info")@@ -1083,7 +1086,7 @@ def parse_config():return config-def load(urlString: str, encoding: str = "utf-8"):+def load(urlString: str, encoding: str = "utf-8") -> str:try:content = urllib.request.urlopen(urlString).read().decode(encoding)except Exception as e:@@ -1092,7 +1095,7 @@ def load(urlString: str, encoding: str = "utf-8"):return content-def configure_pgp(gpg_todo: Todo):+def configure_pgp(gpg_todo: Todo) -> bool:print("Based on your Apache ID we'll lookup your key online\nand through this complete the 'gpg' prerequisite task.\n")gpg_state = gpg_todo.get_state()id = str(input("Please enter your Apache id: (ENTER=skip) "))@@ -1237,7 +1240,7 @@ def configure_pgp(gpg_todo: Todo):return True-def pause(fun: Callable[[], None] | None = None):+def pause(fun: Callable[[], None] | None = None) -> None:if fun:fun()input("\nPress ENTER to continue...")@@ -1245,15 +1248,19 @@ def pause(fun: Callable[[], None] | None = None):class CustomExitItem(ExitItem):@override- def show(self, index: int, available_width: int | None = None):+ def show(self, index: int, available_width: int | None = None) -> Any:return super(CustomExitItem, self).show(index)@override- def get_return(self):+ def get_return(self) -> str:return ""-def main():+templates: Any+lucene_news_file: str+++def main() -> None:global stateglobal dry_runglobal templates@@ -1359,11 +1366,11 @@ script_path = os.path.dirname(os.path.realpath(__file__))os.chdir(script_path)-def git_checkout_folder():+def git_checkout_folder() -> str:return state.get_git_checkout_folder()-def tail_file(file: str, lines: int):+def tail_file(file: str, lines: int) -> None:bufsize = 8192fsize = os.stat(file).st_sizewith open(file) as f:@@ -1383,7 +1390,9 @@ def tail_file(file: str, lines: int):break-def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):+def run_with_log_tail(+ command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None+) -> int | None:fh = sys.stdoutif logfile:logdir = os.path.dirname(logfile)@@ -1398,7 +1407,7 @@ def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str |return rc-def ask_yes_no(text: str):+def ask_yes_no(text: str) -> bool:answer = Nonewhile answer not in ["y", "n"]:answer = str(input("\nQ: %s (y/n): " % text))@@ -1406,16 +1415,16 @@ def ask_yes_no(text: str):return answer == "y"-def abbreviate_line(line: str, width: int):+def abbreviate_line(line: str, width: int) -> str:line = line.rstrip()if len(line) > width:- line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :])+ line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :]) # ty:ignore[invalid-argument-type]else:line = "%s%s" % (line, " " * (width - len(line) + 2))return line-def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False):+def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False) -> None:if not tee:if not stdout:print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end="\r")@@ -1425,7 +1434,7 @@ def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | Noneprint(line.rstrip())-def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):+def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None) -> int | None:doShell = "&&" in command or "&" in command or shell is not Noneif not doShell and not isinstance(command, list):command = shlex.split(command)@@ -1502,15 +1511,15 @@ def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = syreturn rc-def is_windows():+def is_windows() -> bool:return platform.system().startswith("Win")-def is_mac():+def is_mac() -> bool:return platform.system().startswith("Darwin")-def is_linux():+def is_linux() -> bool:return platform.system().startswith("Linux")@@ -1532,7 +1541,7 @@ class Commands(SecretYamlObject):vars: dict[str, Any] | None = None,todo_id: str | None = None,remove_files: list[str] | None = None,- ):+ ) -> None:self.root_folder = root_folderself.commands_text = commands_textself.vars = vars@@ -1549,11 +1558,11 @@ class Commands(SecretYamlObject):@classmethod@override- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Commands":fields = loader.construct_mapping(node, deep=True)return Commands(**cast("dict[str, Any]", fields))- def run(self): # pylint: disable=inconsistent-return-statements # TODO+ def run(self) -> bool | None:root = self.get_root_folder()if self.commands_text:@@ -1667,19 +1676,19 @@ class Commands(SecretYamlObject):print("WARNING: One or more commands failed, you may want to check the logs")return success- def get_root_folder(self):+ def get_root_folder(self) -> str:return cast("str", self.jinjaify(self.root_folder))- def get_commands_text(self):+ def get_commands_text(self) -> str | list[str] | None:return self.jinjaify(self.commands_text)- def get_run_text(self):+ def get_run_text(self) -> str | list[str] | None:return self.jinjaify(self.run_text)- def get_remove_files(self):+ def get_remove_files(self) -> str | list[str] | None:return self.jinjaify(self.remove_files)- def get_vars(self):+ def get_vars(self) -> dict[str, str]:myvars: dict[str, str] = {}if self.vars:for k in self.vars:@@ -1690,7 +1699,7 @@ class Commands(SecretYamlObject):myvars[k] = expand_jinja(val, vars=myvars)return myvars- def jinjaify(self, data: str | list[str] | None, join: bool = False):+ def jinjaify(self, data: str | list[str] | None, join: bool = False) -> str | list[str] | None:if not data:return Nonev = self.get_vars()@@ -1708,7 +1717,7 @@ class Commands(SecretYamlObject):return expand_jinja(data, v)-def abbreviate_homedir(line: str):+def abbreviate_homedir(line: str) -> str:if is_windows():if "HOME" in os.environ:return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%HOME%", line)@@ -1737,7 +1746,7 @@ class Command(SecretYamlObject):redirect: str | None = None,redirect_append: bool | None = None,shell: bool | None = None,- ):+ ) -> None:self.cmd = cmdself.cwd = cwdself.comment = comment@@ -1766,21 +1775,21 @@ class Command(SecretYamlObject):@classmethod@override- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Command":fields = loader.construct_mapping(node, deep=True)return Command(**cast("dict[str, Any]", fields))- def get_comment(self):+ def get_comment(self) -> str:return str(self.jinjaify(self.comment))- def get_redirect(self):+ def get_redirect(self) -> str:return str(self.jinjaify(self.redirect))- def get_cmd(self):+ def get_cmd(self) -> str:return str(self.jinjaify(self.cmd, join=True))- def get_vars(self):- myvars: dict[str, str] = {}+ def get_vars(self) -> dict[str, str]:+ myvars = {}if self.vars:for k in self.vars:val = self.vars[k]@@ -1791,10 +1800,10 @@ class Command(SecretYamlObject):return myvars@override- def __str__(self):+ def __str__(self) -> str:return self.get_cmd()- def jinjaify(self, data: str | list[str] | None, join: bool = False):+ def jinjaify(self, data: str | list[str] | None, join: bool = False) -> str | list[str] | None:if data is None:return Nonev = self.get_vars()@@ -1811,7 +1820,7 @@ class Command(SecretYamlObject):return resreturn expand_jinja(data, v)- def display_cmd(self):+ def display_cmd(self) -> list[str]:lines: list[str] = []if self.comment:if is_windows():@@ -1833,18 +1842,18 @@ class Command(SecretYamlObject):class UserInput(SecretYamlObject):yaml_tag = "!UserInput"- def __init__(self, name: str, prompt: str, type: str | None = None):+ def __init__(self, name: str, prompt: str, type: str | None = None) -> None:self.type = typeself.prompt = promptself.name = name@classmethod@override- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "UserInput":fields = loader.construct_mapping(node, deep=True)return UserInput(**cast("dict[str, Any]", fields))- def run(self, dict: dict[str, Any] | None = None):+ def run(self, dict: dict[str, Any] | None = None) -> str | int | None:correct = Falsewhile not correct:try:@@ -1860,7 +1869,7 @@ class UserInput(SecretYamlObject):return result-def create_ical(_todo: Todo): # pylint: disable=unused-argument+def create_ical(_todo: Todo) -> bool:if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):c = Calendar()e = Event()@@ -1870,7 +1879,7 @@ def create_ical(_todo: Todo): # pylint: disable=unused-argumentc.events.add(e)ics_file = os.path.join(state.get_rc_folder(), "vote_end.ics")with open(ics_file, "w") as my_file:- my_file.writelines(c) # pyright: ignore[reportArgumentType]+ my_file.writelines(iter(c))open_file(ics_file)return True@@ -1890,12 +1899,12 @@ non_working = ()-def vote_close_72h_date():+def vote_close_72h_date() -> datetime:# Voting open at least 72 hours according to ASF policyreturn datetime.now(tz=UTC) + timedelta(hours=73)-def vote_close_72h_holidays():+def vote_close_72h_holidays() -> list[str] | None:days = 0day_offset = -1holidays: list[str] = []@@ -1913,7 +1922,7 @@ def vote_close_72h_holidays():return holidays if len(holidays) > 0 else None-def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argument+def prepare_announce_lucene(_todo: Todo) -> bool:if not os.path.exists(lucene_news_file):lucene_text = expand_jinja("(( template=announce_lucene ))")with open(lucene_news_file, "w") as fp:@@ -1924,7 +1933,7 @@ def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argumentreturn True-def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argument+def check_artifacts_available(_todo: Todo) -> bool:cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")try:load(cdnUrl)@@ -1944,12 +1953,12 @@ def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argumentreturn True-def set_java_home(version: str):+def set_java_home(version: str) -> None:os.environ["JAVA_HOME"] = state.get_java_home_for_version(version)os.environ["JAVACMD"] = state.get_java_cmd_for_version(version)-def load_lines(file: str, from_line: int = 0):+def load_lines(file: str, from_line: int = 0) -> list[str]:if os.path.exists(file):with open(file) as fp:return fp.readlines()[from_line:]diff --git a/dev-tools/scripts/releasedJirasRegex.py b/dev-tools/scripts/releasedJirasRegex.pyindex 0dd5ccd6360..f972a8d2cb7 100755--- a/dev-tools/scripts/releasedJirasRegex.py+++ b/dev-tools/scripts/releasedJirasRegex.py@@ -32,7 +32,7 @@ from scriptutil import Version# does not find Bugzilla bugs or JIRAs not mentioned at the beginning of# bullets or numbered entries.#-def print_released_jiras_regex(version: str, filename: str):+def print_released_jiras_regex(version: str, filename: str) -> None:release_boundary_re = re.compile(r"\s*====*\s+(.*)\s+===")version_re = re.compile(r"%s(?:$|[^-])" % version)bullet_re = re.compile(r"\s*(?:[-*]|\d+\.(?=(?:\s|(?:LUCENE)-)))(.*)")@@ -74,14 +74,14 @@ def print_released_jiras_regex(version: str, filename: str):print()-def read_config():+def read_config() -> argparse.Namespace:parser = argparse.ArgumentParser(description="Prints a regex matching JIRAs fixed in the given version by parsing the given CHANGES.txt file")parser.add_argument("version", type=Version.parse, help="Version of the form X.Y.Z")parser.add_argument("changes", help="CHANGES.txt file to parse")return parser.parse_args()-def main():+def main() -> None:config = read_config()print_released_jiras_regex(config.version, config.changes)diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.pyindex 46ddbfb1600..a889f51c16e 100644--- a/dev-tools/scripts/reproduceJenkinsFailures.py+++ b/dev-tools/scripts/reproduceJenkinsFailures.py@@ -72,7 +72,7 @@ description = dedent("""\defaultIters = 5-def readConfig():+def readConfig() -> argparse.Namespace:parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=description)parser.add_argument("url", metavar="URL", help="Points to the Jenkins log to parse")parser.add_argument("--no-git", dest="useGit", action="store_false", default=True, help='Do not run "git" at all')@@ -80,7 +80,7 @@ def readConfig():return parser.parse_args()-def runOutput(cmd: str):+def runOutput(cmd: str) -> str:print("[repro] %s" % cmd)try:return subprocess.check_output(cmd.split(" "), universal_newlines=True).strip()@@ -89,16 +89,21 @@ def runOutput(cmd: str):# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False-def run(cmd: str, rememberFailure: bool = True):+def run(cmd: str, rememberFailure: bool = True) -> int:global lastFailureCodeprint("[repro] %s" % cmd)- code = os.system(cmd)+ code = os.system(cmd) # ty:ignore[deprecated]if code != 0 and rememberFailure:print("\n[repro] Setting last failure code to %d\n" % code)lastFailureCode = codereturn code+antOptions: str+branchFromLog: str | None+revisionFromLog: str | None++def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:global revisionFromLogglobal branchFromLog@@ -152,7 +157,7 @@ def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:return tests-def prepareWorkspace(useGit: bool, gitRef: str | None):+def prepareWorkspace(useGit: bool, gitRef: str | None) -> None:global gitCheckoutSucceededif useGit:code = run("git fetch")@@ -178,7 +183,7 @@ def prepareWorkspace(useGit: bool, gitRef: str | None):raise RuntimeError('ERROR: "ant clean" failed. See above.')-def groupTestsByModule(tests: dict[str, str]):+def groupTestsByModule(tests: dict[str, str]) -> dict[str, set[str]]:modules: dict[str, set[str]] = {}for dir, _, files in os.walk("."):for file in files:@@ -200,7 +205,7 @@ def groupTestsByModule(tests: dict[str, str]):return modules-def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]):+def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]) -> None:cwd = os.getcwd()testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'for module in modules:@@ -218,7 +223,7 @@ def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]os.chdir(cwd)-def printAndMoveReports(testIters: int, newSubDir: str, location: str):+def printAndMoveReports(testIters: int, newSubDir: str, location: str) -> dict[str, int]:failures: dict[str, int] = {}for start in ("lucene/build", "solr/build"):for dir, _, files in os.walk(start):@@ -245,7 +250,7 @@ def printAndMoveReports(testIters: int, newSubDir: str, location: str):return failures-def getLocalGitBranch():+def getLocalGitBranch() -> str:origGitBranch = runOutput("git rev-parse --abbrev-ref HEAD")if origGitBranch == "HEAD": # In detached HEAD stateorigGitBranch = runOutput("git rev-parse HEAD") # Use the SHA when not on a branch@@ -253,7 +258,7 @@ def getLocalGitBranch():return origGitBranch-def main():+def main() -> None:config = readConfig()tests = fetchAndParseJenkinsLog(config.url, numRetries=2)localGitBranch = Nonediff --git a/dev-tools/scripts/requirements.txt b/dev-tools/scripts/requirements.txtindex 1b834d5b70e..44498c4344c 100644--- a/dev-tools/scripts/requirements.txt+++ b/dev-tools/scripts/requirements.txt@@ -2,6 +2,8 @@Jinja2==3.1.6# parsing and processing of releaseWizard.yamlPyYAML==6.0.3+# types+types-PyYAML==6.0.12.20260518# international holidays in releaseWizardholidays==0.95# calendar processing in releaseWizard@@ -13,6 +15,6 @@ PyGithub==2.9.1# JIRA processing in githubPRsjira==3.10.5# type-checking in "make lint"-basedpyright==1.38.4+ty==0.0.46# linting in "make lint"ruff==0.15.15diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.pyindex 7cbabf22abc..9e608b68484 100644--- a/dev-tools/scripts/scriptutil.py+++ b/dev-tools/scripts/scriptutil.py@@ -28,7 +28,7 @@ from typing import Self, overrideclass Version:- def __init__(self, major: int, minor: int, bugfix: int, prerelease: int):+ def __init__(self, major: int, minor: int, bugfix: int, prerelease: int) -> None:super().__init__()self.major = majorself.minor = minor@@ -39,7 +39,7 @@ class Version:self.constant = "LUCENE_%d_%d_%d" % (self.major, self.minor, self.bugfix)@classmethod- def parse(cls, value: str):+ def parse(cls, value: str) -> "Version":match = re.search(r"(\d+)\.(\d+).(\d+)(.1|.2)?", value)if match is None:raise argparse.ArgumentTypeError("Version argument must be of format x.y.z(.1|.2)?")@@ -48,10 +48,10 @@ class Version:return Version(*parts)@override- def __str__(self):+ def __str__(self) -> str:return self.dot- def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\."):+ def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\.") -> Pattern:if self.is_bugfix_release():pattern = "%s%s%s%s%d" % (self.major, sep, self.minor, sep, self.bugfix - 1)elif self.is_minor_release():@@ -61,30 +61,30 @@ class Version:return re.compile(prefix + "(" + pattern + ")" + suffix)- def is_bugfix_release(self):+ def is_bugfix_release(self) -> bool:return self.bugfix != 0- def is_minor_release(self):+ def is_minor_release(self) -> bool:return self.bugfix == 0 and self.minor != 0- def is_major_release(self):+ def is_major_release(self) -> bool:return self.bugfix == 0 and self.minor == 0- def on_or_after(self, other: Self):+ def on_or_after(self, other: Self) -> bool:return self.major > other.major or (self.major == other.major and (self.minor > other.minor or (self.minor == other.minor and (self.bugfix > other.bugfix or (self.bugfix == other.bugfix and self.prerelease >= other.prerelease)))))- def gt(self, other: Self):+ def gt(self, other: Self) -> bool:return self.major > other.major or (self.major == other.major and self.minor > other.minor) or (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix)- def is_back_compat_with(self, other: Self):+ def is_back_compat_with(self, other: Self) -> bool:if not self.on_or_after(other):raise Exception("Back compat check disallowed for newer version: %s < %s" % (self, other))return other.major + 1 >= self.major-def run(cmd: str, cwd: str | None = None):+def run(cmd: str, cwd: str | None = None) -> str:try:output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)except subprocess.CalledProcessError as e:@@ -93,7 +93,7 @@ def run(cmd: str, cwd: str | None = None):return output.decode("utf-8")-def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None):+def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None) -> bool:infile = open(filename)buffer: list[str] = []@@ -123,7 +123,7 @@ class BranchType(Enum):release = 3-def find_branch_type():+def find_branch_type() -> BranchType:output = subprocess.check_output("git status", shell=True)for line in output.split(b"\n"):if line.startswith(b"On branch "):@@ -141,7 +141,7 @@ def find_branch_type():raise Exception("Cannot run %s on feature branch" % sys.argv[0].rsplit("/", 1)[-1])-def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True):+def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True) -> None:if not quiet:print("Downloading %s" % urlString)startTime = time.time()@@ -164,7 +164,7 @@ def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_print(" %.1f MB in %.2f sec (%.1f MB/sec)" % (sizeMB, t, sizeMB / t))-def attemptDownload(urlString: str, fileName: str):+def attemptDownload(urlString: str, fileName: str) -> None:fIn = urllib.request.urlopen(urlString)fOut = open(fileName, "wb")success = False@@ -184,7 +184,7 @@ def attemptDownload(urlString: str, fileName: str):os.remove(fileName)-def find_current_version():+def find_current_version() -> str:version_prop_re = re.compile(r"version\.base=(.*)")script_path = os.path.dirname(os.path.realpath(__file__))top_level_dir = os.path.join(Path("%s/" % script_path).resolve(), os.path.pardir, os.path.pardir)diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.pyindex 4b1e8799db7..25baa7be113 100755--- a/dev-tools/scripts/smokeTestRelease.py+++ b/dev-tools/scripts/smokeTestRelease.py@@ -47,10 +47,10 @@ BASE_JAVA_VERSION = "25"# tested on Linux and on Cygwin under Windows 7.cygwin = platform.system().lower().startswith("cygwin")-cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else ""+cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else "" # ty:ignore[deprecated]-def unshortenURL(url: str):+def unshortenURL(url: str) -> str:parsed = urllib.parse.urlparse(url)if parsed[0] in ("http", "https"):h = http.client.HTTPConnection(parsed.netloc)@@ -71,7 +71,7 @@ reHREF = re.compile('<a href="(.*?)">(.*?)</a>')FORCE_CLEAN = True-def getHREFs(urlString: str):+def getHREFs(urlString: str) -> list[tuple[str, str]]:# Deref any redirectswhile True:url = urllib.parse.urlparse(urlString)@@ -103,7 +103,7 @@ def getHREFs(urlString: str):return links-def load(urlString: str):+def load(urlString: str) -> str:try:content = urllib.request.urlopen(urlString).read().decode("utf-8")except Exception as e:@@ -112,14 +112,14 @@ def load(urlString: str):return content-def noJavaPackageClasses(desc: str, file: str):+def noJavaPackageClasses(desc: str, file: str) -> None:with zipfile.ZipFile(file) as z2:for name2 in z2.namelist():if name2.endswith(".class") and (name2.startswith("java/") or name2.startswith("javax/")):raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))-def decodeUTF8(bytes: bytes):+def decodeUTF8(bytes: bytes) -> str:return codecs.getdecoder("UTF-8")(bytes)[0]@@ -128,7 +128,7 @@ NOTICE_FILE_NAME = "META-INF/NOTICE.txt"LICENSE_FILE_NAME = "META-INF/LICENSE.txt"-def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):+def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str) -> None:with zipfile.ZipFile(jarFile, "r") as z:for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME):try:@@ -186,11 +186,11 @@ def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):raise RuntimeError("%s: %s contents doesn't match main LICENSE.txt" % (desc, LICENSE_FILE_NAME))-def normSlashes(path: str):+def normSlashes(path: str) -> str:return path.replace(os.sep, "/")-def checkAllJARs(topDir: str, gitRevision: str, version: str):+def checkAllJARs(topDir: str, gitRevision: str, version: str) -> None:print(" verify JAR metadata/identity/no javax.* or java.* classes...")for root, _, files in os.walk(topDir):normRoot = normSlashes(root)@@ -205,7 +205,7 @@ def checkAllJARs(topDir: str, gitRevision: str, version: str):checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)-def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str):+def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str) -> None:print(" test basics...")ents = getDirEntries(urlString)artifact = None@@ -305,7 +305,7 @@ def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFilprint(" GPG: %s" % line.strip())-def testChanges(version: str, changesURLString: str):+def testChanges(version: str, changesURLString: str) -> None:print(" check changes HTML...")changesURL = Nonefor text, subURL in getDirEntries(changesURLString):@@ -319,7 +319,7 @@ def testChanges(version: str, changesURLString: str):checkChangesContent(s, version, changesURL, True)-def testChangesText(dir: str, version: str):+def testChangesText(dir: str, version: str) -> None:"Checks all CHANGES.txt under this dir."for root, _, files in os.walk(dir):# NOTE: O(N) but N should be smallish:@@ -334,7 +334,7 @@ reUnderbarNotDashHTML = re.compile(r"<li>(\s*(LUCENE)_\d\d\d\d+)")reUnderbarNotDashTXT = re.compile(r"\s+((LUCENE)_\d\d\d\d+)", re.MULTILINE)-def checkChangesContent(s: str, version: str, name: str, isHTML: bool):+def checkChangesContent(s: str, version: str, name: str, isHTML: bool) -> None:currentVersionTuple = versionToTuple(version, name)if isHTML and s.find("Release %s" % version) == -1:@@ -371,7 +371,7 @@ def checkChangesContent(s: str, version: str, name: str, isHTML: bool):release = text[8:].strip()seenText.clear()releaseTuple = versionToTuple(release, name)- if releaseTuple > currentVersionTuple:+ if releaseTuple > currentVersionTuple: # ty:ignore[unsupported-operator]raise RuntimeError("Future release %s is greater than %s in %s" % (release, version, name))if id in seenIDs:raise RuntimeError('%s has duplicate section "%s" under release "%s"' % (name, text, release))@@ -384,7 +384,7 @@ def checkChangesContent(s: str, version: str, name: str, isHTML: bool):reVersion = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?", re.IGNORECASE)-def versionToTuple(version: str, name: str):+def versionToTuple(version: str, name: str) -> tuple[int | str | Any, ...]:versionMatch = reVersion.match(version)if versionMatch is None:raise RuntimeError("Version %s in %s cannot be parsed" % (version, name))@@ -405,7 +405,7 @@ def versionToTuple(version: str, name: str):reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + "|(?:\\\\.|[^\"'\\s])*" + r"|'(?:\\'|[^'])*')" + r'|(/(?:\\.|[^"\'\s])*)' + r'|("/(?:\\.|[^"])*")' + r"|('/(?:\\.|[^'])*')")-def unix2win(matchobj: re.Match[str]):+def unix2win(matchobj: re.Match[str]) -> str:if matchobj.group(1) is not None:return cygwinWindowsRoot + matchobj.group()if matchobj.group(2) is not None:@@ -415,7 +415,7 @@ def unix2win(matchobj: re.Match[str]):return matchobj.group()-def cygwinifyPaths(command: str):+def cygwinifyPaths(command: str) -> str:# The problem: Native Windows applications running under Cygwin can't# handle Cygwin's Unix-style paths. However, environment variable# values are automatically converted, so only paths outside of@@ -426,7 +426,7 @@ def cygwinifyPaths(command: str):return command-def printFileContents(fileName: str):+def printFileContents(fileName: str) -> None:# Assume log file was written in system's default encoding, but# even if we are wrong, we replace errors ... the ASCII chars# (which is what we mostly care about eg for the test seed) should@@ -443,17 +443,17 @@ def printFileContents(fileName: str):print()-def run(command: str, logFile: str):+def run(command: str, logFile: str) -> None:if cygwin:command = cygwinifyPaths(command)- if os.system("%s > %s 2>&1" % (command, logFile)):+ if os.system("%s > %s 2>&1" % (command, logFile)): # ty:ignore[deprecated]logPath: str = str(Path(logFile).resolve())print('\ncommand "%s" failed:' % command)printFileContents(logFile)raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))-def verifyDigests(artifact: str, urlString: str, tmpDir: str):+def verifyDigests(artifact: str, urlString: str, tmpDir: str) -> None:print(" verify sha512 digest")sha512Expected, t = load(urlString + ".sha512").strip().split()if t != "*" + artifact:@@ -472,7 +472,7 @@ def verifyDigests(artifact: str, urlString: str, tmpDir: str):raise RuntimeError("SHA512 digest mismatch for %s: expected %s but got %s" % (artifact, sha512Expected, sha512Actual))-def getDirEntries(urlString: str):+def getDirEntries(urlString: str) -> list[tuple[str, str]]:if urlString.startswith("file:/") and not urlString.startswith("file://"):# stupid bogus ant URIurlString = "file:///" + urlString[6:]@@ -498,7 +498,7 @@ def getDirEntries(urlString: str):raise RuntimeError("could not enumerate %s" % (urlString))-def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str):+def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str) -> str:destDir = "%s/unpack" % tmpDirif os.path.exists(destDir):shutil.rmtree(destDir)@@ -526,7 +526,7 @@ LUCENE_NOTICE = NoneLUCENE_LICENSE = None-def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):+def is_in_list(in_folder: list[str], files: list[str], indent: int = 4) -> None:for fileName in files:print("%sChecking %s" % (" " * indent, fileName))found = False@@ -538,7 +538,7 @@ def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):raise RuntimeError('file "%s" is missing' % fileName)-def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str):+def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str) -> None:global LUCENE_NOTICEglobal LUCENE_LICENSE@@ -633,13 +633,13 @@ def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str,if isSrc:print(" make sure no JARs/WARs in src dist...")- lines = os.popen("find . -name \\*.jar -not -name \\*-api.jar").readlines()+ lines = os.popen("find . -name \\*.jar -not -name \\*-api.jar").readlines() # ty:ignore[deprecated]if len(lines) != 0:print(" FAILED:")for line in lines:print(" %s" % line.strip())raise RuntimeError("source release has JARs...")- lines = os.popen("find . -name \\*.war").readlines()+ lines = os.popen("find . -name \\*.war").readlines() # ty:ignore[deprecated]if len(lines) != 0:print(" FAILED:")for line in lines:@@ -678,7 +678,7 @@ def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str,testChangesText(".", version)-def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str):+def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str) -> None:if os.path.exists("index"):shutil.rmtree("index") # nuke any index from any previous iteration@@ -727,11 +727,11 @@ def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdraise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))-def removeTrailingZeros(version: str):+def removeTrailingZeros(version: str) -> str:return re.sub(r"(\.0)*$", "", version)-def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str):+def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str) -> None:print(" download artifacts")artifacts: list[str] = []artifactsURL = "%s/lucene/maven/org/apache/lucene/" % baseURL@@ -753,7 +753,7 @@ def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigncheckAllJARs("%s/maven/org/apache/lucene" % tmpDir, gitRevision, version)-def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):+def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str) -> list[str]:distribution = "lucene-%s.tgz" % versionif not os.path.exists("%s/%s" % (tmpDir, distribution)):distURL = "%s/lucene/%s" % (baseURL, distribution)@@ -767,13 +767,13 @@ def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):print(" unpack %s..." % distribution)unpackLogFile = "%s/unpack-%s-getBinaryDistFiles.log" % (tmpDir, distribution)run("tar xzf %s/%s" % (tmpDir, distribution), unpackLogFile)- distributionFiles: list[str] = []+ distributionFiles = []for root, _, files in os.walk(destDir):distributionFiles.extend([os.path.join(root, file) for file in files])return distributionFiles-def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):+def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str) -> None:print(" check for javadoc and sources artifacts...")for artifact in artifacts:if artifact.endswith(version + ".jar"):@@ -785,8 +785,8 @@ def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):raise RuntimeError("missing: %s" % sourcesJar)-def getZipFileEntries(fileName: str):- entries: list[str] = []+def getZipFileEntries(fileName: str) -> list[str]:+ entries = []with zipfile.ZipFile(fileName) as zf:for zi in zf.infolist():entries.append(zi.filename)@@ -795,7 +795,7 @@ def getZipFileEntries(fileName: str):return entries-def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str):+def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str) -> None:print(" verify that Maven artifacts are same as in the binary distribution...")reJarWar = re.compile(r"%s\.[wj]ar$" % version) # exclude *-javadoc.jar and *-sources.jardistFilenames: dict[str, str] = dict()@@ -812,7 +812,7 @@ def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], verraise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))-def verifyMavenDigests(artifacts: list[str]):+def verifyMavenDigests(artifacts: list[str]) -> None:print(" verify Maven artifacts' md5/sha1 digests...")reJarWarPom = re.compile(r"\.(?:[wj]ar|pom)$")for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:@@ -842,7 +842,7 @@ def verifyMavenDigests(artifacts: list[str]):raise RuntimeError("SHA1 digest mismatch for %s: expected %s but got %s" % (artifactFile, sha1Expected, sha1Actual))-def getPOMcoordinate(treeRoot: ET.Element):+def getPOMcoordinate(treeRoot: ET.Element) -> tuple[str, str, str, str]:namespace = "{http://maven.apache.org/POM/4.0.0}"groupId = treeRoot.find("%sgroupId" % namespace)if groupId is None:@@ -872,7 +872,7 @@ def getPOMcoordinate(treeRoot: ET.Element):return groupId, artifactId, packaging, version-def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):+def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str) -> None:print(" verify maven artifact sigs", end=" ")# Set up clean gpg world; import keys file:@@ -904,14 +904,14 @@ def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):print()-def print_warnings_in_file(file: str):+def print_warnings_in_file(file: str) -> None:with open(file) as f:for line in f.readlines():if line.lower().find("warning") != -1 and line.find("WARNING: This key is not certified with a trusted signature") == -1 and line.find("WARNING: using insecure memory") == -1:print(" GPG: %s" % line.strip())-def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):+def verifyPOMperBinaryArtifact(artifacts: list[str], version: str) -> None:print(" verify that each binary artifact has a deployed POM...")reBinaryJarWar = re.compile(r"%s\.[jw]ar$" % re.escape(version))for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:@@ -920,7 +920,7 @@ def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):raise RuntimeError("missing: POM for %s" % artifact)-def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):+def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str) -> None:"""Verify that each POM's coordinate (drawn from its content) matchesits filepath, and verify that the corresponding artifact exists."""@@ -937,7 +937,7 @@ def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):raise RuntimeError("Missing corresponding .%s artifact for POM %s" % (packaging, POM))-def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None):+def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None) -> None:exclude: set[str] = exclusions or set()for text, subURL in getDirEntries(urlString):if text not in exclude:@@ -953,8 +953,11 @@ def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusionssys.stdout.write(".")-def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]):- def _make_runner(java_home: str, is_base_version: bool = False):+JavaConfig = namedtuple("JavaConfig", "run_java java_home run_alt_javas alt_java_homes alt_java_versions")+++def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]) -> JavaConfig:+ def _make_runner(java_home: str, is_base_version: bool = False) -> tuple[Callable[[str, str], None], str]:if cygwin:java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode("utf-8").strip()cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (java_home, java_home, java_home)@@ -972,7 +975,7 @@ def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str])elif int(actual_version) < int(BASE_JAVA_VERSION):parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))- def run_java(cmd: str, logfile: str):+ def run_java(cmd: str, logfile: str) -> None:run("%s; %s" % (cmd_prefix, cmd), logfile)return run_java, actual_version@@ -989,15 +992,14 @@ def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str])run_alt_javas.append(run_alt_java)alt_java_versions.append(version)- jc = namedtuple("JavaConfig", "run_java java_home run_alt_javas alt_java_homes alt_java_versions")- return jc(run_java, java_home, run_alt_javas, alt_java_homes, alt_java_versions)+ return JavaConfig(run_java, java_home, run_alt_javas, alt_java_homes, alt_java_versions)version_re = re.compile(r"(\d+\.\d+\.\d+(-ALPHA|-BETA)?)")revision_re = re.compile(r"rev-([a-f\d]+)")-def parse_config():+def parse_config() -> argparse.Namespace:epilogue = textwrap.dedent("""Example usage:python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/lucene/lucene-9.0.0-RC1-rev-c7510a0...@@ -1053,7 +1055,7 @@ reVersion1 = re.compile(r"\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<", re.IGNORECASreVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.", re.IGNORECASE)-def getAllLuceneReleases():+def getAllLuceneReleases() -> list[tuple[int, ...]]:s = load("https://archive.apache.org/dist/lucene/java")releases: set[tuple[int, ...]] = set()@@ -1074,7 +1076,7 @@ def getAllLuceneReleases():return releaseList-def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str):+def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str) -> None:print(" find all past Lucene releases...")allReleases = getAllLuceneReleases()# for tup in allReleases:@@ -1146,7 +1148,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str)print(" success!")-def main():+def main() -> None:c = parse_config()# Pick <major>.<minor> part of version and require script to be from same branch@@ -1160,7 +1162,7 @@ def main():smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, " ".join(c.test_args), downloadOnly=c.download_only)-def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False):+def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False) -> None:startTime = datetime.datetime.now()# Tests annotated @Nightly are more resource-intensive but often cover