Skip to content

Apache Lucene open-source search software

build: add missing type-hints to dev-tools and switch to ty

dependabot[bot]409fcc15829d
diff --git a/dev-tools/scripts/Makefile b/dev-tools/scripts/Makefile
index 44fe192a85e..d94559b9323 100644
--- a/dev-tools/scripts/Makefile
+++ b/dev-tools/scripts/Makefile
@@ -20,13 +20,13 @@ PYTHON?=python3
VENV=${PWD}/.venv
# don't behave strangely if these files exist
-.PHONY: lint format reformat autofix ruff ruff-fix pyright env clean smoke-test
+.PHONY: lint format reformat autofix ruff ruff-fix ty env clean smoke-test
# list of directories we check
SOURCES=$(wildcard *.py)
# check formatting, linting, and types
-lint: format ruff pyright
+lint: format ruff ty
# check all formatting
format: env
@@ -56,9 +56,9 @@ ruff-fix: env
$(VENV)/bin/ruff check --fix $(SOURCES)
# checks types
-pyright: env
- # type-check sources with basedpyright
- $(VENV)/bin/basedpyright $(SOURCES)
+ty: env
+ # type-check sources with ty
+ $(VENV)/bin/ty check $(SOURCES)
# runs smoketester against a release
smoke-test: env
diff --git a/dev-tools/scripts/addBackcompatIndexes.py b/dev-tools/scripts/addBackcompatIndexes.py
index fe40ed5ebeb..14791407db3 100755
--- a/dev-tools/scripts/addBackcompatIndexes.py
+++ b/dev-tools/scripts/addBackcompatIndexes.py
@@ -34,7 +34,7 @@ import urllib.request
import scriptutil
-def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str):
+def create_and_add_index(source: str, indextype: str, index_version: scriptutil.Version, current_version: scriptutil.Version, temp_dir: str) -> None:
if not current_version.is_back_compat_with(index_version):
prefix = "unsupported"
else:
@@ -79,7 +79,7 @@ def create_and_add_index(source: str, indextype: str, index_version: scriptutil.
print("done")
-def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version):
+def update_backcompat_tests(index_version: scriptutil.Version, current_version: scriptutil.Version) -> None:
print(" adding new indexes to backcompat tests...", end="", flush=True)
module = "lucene/backward-codecs"
@@ -91,12 +91,12 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:
strip_dash_suffix_re = re.compile(r"-.*")
- def find_version(x: str):
+ def find_version(x: str) -> scriptutil.Version:
x = x.strip()
x = re.sub(strip_dash_suffix_re, "", x) # remove the -suffix if any
return scriptutil.Version.parse(x)
- def edit(buffer: list[str], _match: re.Match[str], line: str):
+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool:
v = find_version(line)
changed = False
if v.on_or_after(index_version):
@@ -106,7 +106,7 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:
buffer.append(line)
return changed
- def append(buffer: list[str], changed: bool):
+ def append(buffer: list[str], changed: bool) -> bool:
if changed:
return changed
if not buffer[len(buffer) - 1].endswith("\n"):
@@ -118,13 +118,13 @@ def update_backcompat_tests(index_version: scriptutil.Version, current_version:
print("done" if changed else "uptodate")
-def check_backcompat_tests():
+def check_backcompat_tests() -> None:
print(" checking backcompat tests...", end="", flush=True)
scriptutil.run("./gradlew -p lucene/backward-codecs test --tests TestGenerateBwcIndices")
print("ok")
-def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str):
+def download_from_cdn(version: scriptutil.Version, remotename: str, localname: str) -> bool:
url = "http://dlcdn.apache.org/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -135,7 +135,7 @@ def download_from_cdn(version: scriptutil.Version, remotename: str, localname: s
raise e
-def download_from_archives(version: scriptutil.Version, remotename: str, localname: str):
+def download_from_archives(version: scriptutil.Version, remotename: str, localname: str) -> bool:
url = "http://archive.apache.org/dist/lucene/java/%s/%s" % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
@@ -146,7 +146,7 @@ def download_from_archives(version: scriptutil.Version, remotename: str, localna
raise e
-def download_release(version: scriptutil.Version, temp_dir: str, force: bool):
+def download_release(version: scriptutil.Version, temp_dir: str, force: bool) -> str:
print(" downloading %s source release..." % version, end="", flush=True)
source = os.path.join(temp_dir, "lucene-%s" % version)
if os.path.exists(source):
@@ -169,7 +169,7 @@ def download_release(version: scriptutil.Version, temp_dir: str, force: bool):
return source
-def read_config():
+def read_config() -> argparse.Namespace:
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""\
@@ -186,7 +186,7 @@ http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
return c
-def main():
+def main() -> None:
c = read_config()
if not os.path.exists(c.temp_dir):
Path(c.temp_dir).mkdir(parents=True)
diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py
index 997a8a96a0f..cdf1f9ec2f6 100755
--- a/dev-tools/scripts/addVersion.py
+++ b/dev-tools/scripts/addVersion.py
@@ -25,11 +25,11 @@ from configparser import ConfigParser, ExtendedInterpolation
from scriptutil import Version, find_branch_type, find_current_version, run, update_file
-def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]):
+def update_changes(filename: str, new_version: Version, init_changes: str, headers: list[str]) -> None:
print(" adding new section to %s..." % filename, end="", flush=True)
matcher = re.compile(r"\d+\.\d+\.\d+\s+===")
- def edit(buffer: list[str], _match: re.Match[str], line: str):
+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:
if new_version.dot in line:
return None
match = new_version.previous_dot_matcher.search(line)
@@ -45,14 +45,14 @@ def update_changes(filename: str, new_version: Version, init_changes: str, heade
print("done" if changed else "uptodate")
-def add_constant(new_version: Version, deprecate: bool):
+def add_constant(new_version: Version, deprecate: bool) -> None:
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
print(" adding constant %s..." % new_version.constant, end="", flush=True)
constant_prefix = "public static final Version LUCENE_"
matcher = re.compile(constant_prefix)
prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep="_")
- def ensure_deprecated(buffer: list[str]):
+ def ensure_deprecated(buffer: list[str]) -> None:
last = buffer[-1]
if last.strip() != "@Deprecated":
spaces = " " * (len(last) - len(last.lstrip()) - 1)
@@ -61,7 +61,7 @@ def add_constant(new_version: Version, deprecate: bool):
del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
buffer.append(("{0} * @deprecated ({1}) Use latest\n" + "{0} */\n" + "{0}@Deprecated\n").format(spaces, new_version))
- def buffer_constant(buffer: list[str], line: str):
+ def buffer_constant(buffer: list[str], line: str) -> None:
spaces = " " * (len(line) - len(line.lstrip()))
buffer.append(("\n{0}/**\n" + "{0} * Match settings and bugs in Lucene's {1} release.\n").format(spaces, new_version))
if deprecate:
@@ -76,7 +76,7 @@ def add_constant(new_version: Version, deprecate: bool):
class Edit:
found = -1
- def __call__(self, buffer: list[str], _match: re.Match[str], line: str):
+ def __call__(self, buffer: list[str], _match: re.Match[str], line: str) -> bool | None:
if new_version.constant in line:
return None # constant already exists
# outer match is just to find lines declaring version constants
@@ -102,11 +102,11 @@ def add_constant(new_version: Version, deprecate: bool):
print("done" if changed else "uptodate")
-def update_build_version(new_version: Version):
+def update_build_version(new_version: Version) -> None:
print(" changing baseVersion...", end="", flush=True)
filename = "build-options.properties"
- def edit(buffer: list[str], _match: re.Match[str], line: str):
+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:
if new_version.dot in line:
return None
buffer.append("version.base=" + new_version.dot + "\n")
@@ -117,12 +117,12 @@ def update_build_version(new_version: Version):
print("done" if changed else "uptodate")
-def update_latest_constant(new_version: Version):
+def update_latest_constant(new_version: Version) -> None:
print(" changing Version.LATEST to %s..." % new_version.constant, end="", flush=True)
filename = "lucene/core/src/java/org/apache/lucene/util/Version.java"
matcher = re.compile("public static final Version LATEST")
- def edit(buffer: list[str], _match: re.Match[str], line: str):
+ def edit(buffer: list[str], _match: re.Match[str], line: str) -> bool | None:
if new_version.constant in line:
return None
buffer.append(line.rpartition("=")[0] + ("= %s;\n" % new_version.constant))
@@ -132,17 +132,17 @@ def update_latest_constant(new_version: Version):
print("done" if changed else "uptodate")
-def onerror(x: Exception):
+def onerror(x: Exception) -> None:
raise x
-def check_lucene_version_tests():
+def check_lucene_version_tests() -> None:
print(" checking lucene version tests...", end="", flush=True)
run("./gradlew -p lucene/core test --tests TestVersion")
print("ok")
-def read_config(current_version: Version):
+def read_config(current_version: Version) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Add a new version to CHANGES, to Version.java and build.gradle files")
parser.add_argument("version", type=Version.parse)
newconf = parser.parse_args()
@@ -156,14 +156,14 @@ def read_config(current_version: Version):
# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files
-def parse_properties_file(filename: str):
+def parse_properties_file(filename: str) -> dict[str, str]:
contents = open(filename, encoding="ISO-8859-1").read().replace("%", "%%") # Escape interpolation metachar
parser = ConfigParser(interpolation=ExtendedInterpolation()) # Handle ${property-name} interpolation
parser.read_string("[DUMMY_SECTION]\n" + contents) # Add required section
return dict(parser.items("DUMMY_SECTION"))
-def main():
+def main() -> None:
if not os.path.exists("build-options.properties"):
sys.exit("Tool must be run from the root of a source checkout.")
current_version = Version.parse(find_current_version())
diff --git a/dev-tools/scripts/buildAndPushRelease.py b/dev-tools/scripts/buildAndPushRelease.py
index e028120eaf5..196c5fb9cb8 100755
--- a/dev-tools/scripts/buildAndPushRelease.py
+++ b/dev-tools/scripts/buildAndPushRelease.py
@@ -34,21 +34,21 @@ LOG = "/tmp/release.log"
dev_mode = False
-def log(msg: str):
+def log(msg: str) -> None:
f = open(LOG, mode="ab")
f.write(msg.encode("utf-8"))
f.close()
-def run(command: str):
+def run(command: str) -> None:
log("\n\n%s: RUN: %s\n" % (datetime.datetime.now(), command))
- if os.system("%s >> %s 2>&1" % (command, LOG)):
+ if os.system("%s >> %s 2>&1" % (command, LOG)): # ty:ignore[deprecated]
msg = " FAILED: %s [see log %s]" % (command, LOG)
print(msg)
raise RuntimeError(msg)
-def runAndSendGPGPassword(command: str, password: str):
+def runAndSendGPGPassword(command: str, password: str) -> None:
p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
f = open(LOG, "ab")
while True:
@@ -76,7 +76,7 @@ def runAndSendGPGPassword(command: str, password: str):
raise RuntimeError(msg)
-def load(urlString: str, encoding: str = "utf-8"):
+def load(urlString: str, encoding: str = "utf-8") -> str:
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -85,9 +85,9 @@ def load(urlString: str, encoding: str = "utf-8"):
return content
-def getGitRev():
+def getGitRev() -> str:
if not dev_mode:
- status = os.popen("git status").read().strip()
+ status = os.popen("git status").read().strip() # ty:ignore[deprecated]
if "nothing to commit, working directory clean" not in status and "nothing to commit, working tree clean" not in status:
raise RuntimeError("git clone is dirty:\n\n%s" % status)
if "Your branch is ahead of" in status:
@@ -95,10 +95,10 @@ def getGitRev():
print(" git clone is clean")
else:
print(" Ignoring dirty git clone due to dev-mode")
- return os.popen("git rev-parse HEAD").read().strip()
+ return os.popen("git rev-parse HEAD").read().strip() # ty:ignore[deprecated]
-def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False):
+def prepare(root: str, version: str, pause_before_sign: bool, gpg_key_id: str | None, gpg_password: str | None, gpg_home: str | None = None, sign_gradle: bool = False) -> str:
print()
print("Prepare release...")
if os.path.exists(LOG):
@@ -163,7 +163,7 @@ reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<", re.IGNOREC
reDoapRevision = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?", re.IGNORECASE)
-def checkDOAPfiles(version: str):
+def checkDOAPfiles(version: str) -> None:
# In Lucene DOAP file, verify presence of all releases less than the one being produced.
errorMessages: list[str] = []
for product in ["lucene"]:
@@ -196,7 +196,7 @@ def checkDOAPfiles(version: str):
raise RuntimeError("\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)" % "\n".join(errorMessages))
-def normalizeVersion(tup: tuple[str, ...]):
+def normalizeVersion(tup: tuple[str, ...]) -> str:
suffix = ""
if tup[-1] is not None and tup[-1].lower() == "-alpha":
tup = tup[: (len(tup) - 1)]
@@ -211,7 +211,7 @@ def normalizeVersion(tup: tuple[str, ...]):
return ".".join(tup) + suffix
-def pushLocal(version: str, root: str, rcNum: int, localDir: str):
+def pushLocal(version: str, root: str, rcNum: int, localDir: str) -> str:
print("Push local [%s]..." % localDir)
Path(localDir).mkdir(parents=True)
@@ -240,11 +240,11 @@ def pushLocal(version: str, root: str, rcNum: int, localDir: str):
return "file://%s/%s" % (Path(localDir).resolve(), dir)
-def read_version(_path: str):
+def read_version(_path: str) -> str:
return scriptutil.find_current_version()
-def parse_config():
+def parse_config() -> argparse.Namespace:
epilogue = textwrap.dedent("""
Example usage for a Release Manager:
python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
@@ -300,7 +300,7 @@ def parse_config():
cwd = os.getcwd()
os.chdir(config.root)
config.root = os.getcwd() # Absolutize root dir
- if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2:
+ if os.system("git rev-parse") or len([d for d in ("dev-tools", "lucene") if os.path.isdir(d)]) != 2: # ty:ignore[deprecated]
parser.error('Root path "%s" is not a valid lucene checkout' % config.root)
os.chdir(cwd)
global LOG
@@ -314,12 +314,12 @@ def parse_config():
return config
-def check_cmdline_tools(): # Fail fast if there are cmdline tool problems
- if os.system("git --version >/dev/null 2>/dev/null"):
+def check_cmdline_tools() -> None: # Fail fast if there are cmdline tool problems
+ if os.system("git --version >/dev/null 2>/dev/null"): # ty:ignore[deprecated]
raise RuntimeError('"git --version" returned a non-zero exit code.')
-def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):
+def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None) -> None:
if gpgKeyID is not None:
print(" Verify your gpg key is in the main KEYS file")
if local_keys is not None:
@@ -364,7 +364,7 @@ def check_key_in_keys(gpgKeyID: str | None, local_keys: str | None):
exit(2)
-def resolve_gpghome():
+def resolve_gpghome() -> str | None:
for p in [
# Linux, macos
os.path.join(os.path.expanduser("~"), ".gnupg"),
@@ -377,7 +377,7 @@ def resolve_gpghome():
return None
-def main():
+def main() -> None:
check_cmdline_tools()
c = parse_config()
diff --git a/dev-tools/scripts/create_line_file_docs.py b/dev-tools/scripts/create_line_file_docs.py
index 856ee9e10ee..f8be5df594f 100644
--- a/dev-tools/scripts/create_line_file_docs.py
+++ b/dev-tools/scripts/create_line_file_docs.py
@@ -31,7 +31,7 @@ DEBUG = False
TARGET_DOC_CHARS = 1024
-def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int):
+def compress_with_seek_points(file_name_in: str, file_name_out: str, num_seek_points: int) -> None:
bytes_per_chunk = os.path.getsize(file_name_in) / num_seek_points
seek_points: list[int] = []
@@ -82,7 +82,7 @@ re_next_non_word_character = re.compile(r"\W", re.UNICODE)
EUROPARL_V7_URL = "https://www.statmt.org/europarl/v7/europarl.tgz"
-def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str):
+def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body_string: str) -> int:
"""Splits docs into smallish (~1 KB) sized docs, repeating same title and date"""
doc_count = 0
while len(body_string) > 0:
@@ -107,7 +107,7 @@ def split_docs(all_out: TextIOWrapper, title_string: str, date_string: str, body
return doc_count
-def sample_europarl():
+def sample_europarl() -> None:
# download europarl.tgz v7, if not already here (in cwd):
file_name = "europarl.tgz"
if not os.path.exists(file_name):
diff --git a/dev-tools/scripts/diff_lucene_changes.py b/dev-tools/scripts/diff_lucene_changes.py
index 4d8e5121f39..df65ca97288 100644
--- a/dev-tools/scripts/diff_lucene_changes.py
+++ b/dev-tools/scripts/diff_lucene_changes.py
@@ -31,7 +31,7 @@ to users and avoid future annoying conflicts on backport.
# e.g. python3 -u diff_lucene_changes.py branch_9_9 main 9.9.0
-def get_changes_url(branch_name: str):
+def get_changes_url(branch_name: str) -> str:
if os.path.isdir(branch_name):
url = f"file://{branch_name}/lucene/CHANGES.txt"
else:
@@ -40,13 +40,13 @@ def get_changes_url(branch_name: str):
return url
-def extract_release_section(changes_txt: bytes, release_name: str):
+def extract_release_section(changes_txt: bytes, release_name: str) -> bytes:
match = re.search(f"=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$", changes_txt.decode("utf-8"), re.MULTILINE | re.DOTALL)
assert match
return match.group(1).encode("utf-8")
-def main():
+def main() -> None:
if len(sys.argv) < 3 or len(sys.argv) > 5:
print("\nUsage: python3 -u dev-tools/scripts/diff_lucene_changes.py <branch1-or-local-clone> <branch2-or-local-clone> <release-name> [diff-commandline-extras]\n")
print(' e.g.: python3 -u dev-tools/scripts/diff_lucene_changes.py branch_9_9 /l/trunk 9.9.0 "-w"\n')
diff --git a/dev-tools/scripts/githubPRs.py b/dev-tools/scripts/githubPRs.py
index b6d425c17bf..efa93d5cfcd 100755
--- a/dev-tools/scripts/githubPRs.py
+++ b/dev-tools/scripts/githubPRs.py
@@ -20,6 +20,7 @@ issue number in title, and the ones where the linked JIRA is already closed
import os
import sys
+from argparse import Namespace
sys.path.append(os.path.dirname(__file__))
import argparse
@@ -35,7 +36,7 @@ if TYPE_CHECKING:
from github.PullRequest import PullRequest
-def read_config():
+def read_config() -> Namespace:
parser = argparse.ArgumentParser(description="Find open Pull Requests that need attention")
parser.add_argument("--json", action="store_true", default=False, help="Output as json")
parser.add_argument("--html", action="store_true", default=False, help="Output as html")
@@ -44,14 +45,12 @@ def read_config():
return newconf
-def out(text: str):
- global conf
+def out(conf: Namespace, text: str) -> None:
if not (conf.json or conf.html):
print(text)
-def make_html(dict: dict[Any, Any]):
- global conf
+def make_html(dict: dict[Any, Any]) -> str:
template = Environment(loader=BaseLoader()).from_string("""
<h1>Lucene Github PR report</h1>
@@ -74,8 +73,7 @@ def make_html(dict: dict[Any, Any]):
return template.render(dict)
-def main():
- global conf
+def main() -> None:
conf = read_config()
token = conf.token if conf.token is not None else None
if token:
@@ -86,9 +84,9 @@ def main():
result: dict[str, Any] = {}
repo = gh.get_repo("apache/lucene")
open_prs = repo.get_pulls(state="open")
- out("Lucene Github PR report")
- out("============================")
- out("Number of open Pull Requests: %s" % open_prs.totalCount)
+ out(conf, "Lucene Github PR report")
+ out(conf, "============================")
+ out(conf, "Number of open Pull Requests: %s" % open_prs.totalCount)
result["open_count"] = open_prs.totalCount
lack_jira = list(filter(lambda x: not re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
@@ -97,11 +95,11 @@ def main():
for pr in lack_jira:
lack_jira_list.append({"title": pr.title, "number": pr.number, "user": pr.user.login, "created": pr.created_at.strftime("%Y-%m-%d")})
result["no_jira"] = lack_jira_list
- out("\nPRs lacking JIRA reference in title")
+ out(conf, "\nPRs lacking JIRA reference in title")
for pr in lack_jira_list:
- out(" #%s: %s %s (%s)" % (pr["number"], pr["created"], pr["title"], pr["user"]))
+ out(conf, " #%s: %s %s (%s)" % (pr["number"], pr["created"], pr["title"], pr["user"]))
- out("\nOpen PRs with a resolved JIRA")
+ out(conf, "\nOpen PRs with a resolved JIRA")
has_jira = list(filter(lambda x: re.match(r".*\b(LUCENE)-\d{3,6}\b", x.title), open_prs))
issue_ids: list[str] = []
@@ -136,7 +134,7 @@ def main():
closed_jiras.sort(key=lambda r: r["pr_number"], reverse=True)
for issue in closed_jiras:
- out(" #%s: %s %s %s: %s (%s)" % (issue["pr_number"], issue["status"], issue["resolution_date"], issue["issue_key"], issue["issue_summary"], issue["assignee"]))
+ out(conf, " #%s: %s %s %s: %s (%s)" % (issue["pr_number"], issue["status"], issue["resolution_date"], issue["issue_key"], issue["issue_summary"], issue["assignee"]))
result["closed_jira_count"] = len(resolved_jiras)
result["closed_jira"] = closed_jiras
diff --git a/dev-tools/scripts/pyproject.toml b/dev-tools/scripts/pyproject.toml
index c19b333c397..ab69b478502 100644
--- a/dev-tools/scripts/pyproject.toml
+++ b/dev-tools/scripts/pyproject.toml
@@ -1,26 +1,16 @@
-[tool.pyright]
-pythonVersion = "3.12"
-venvPath = "."
-venv = ".venv"
-typeCheckingMode = "strict"
-reportCallInDefaultInitializer = "error"
-reportImplicitStringConcatenation = "error"
-reportImportCycles = "error"
-reportImplicitOverride = "error"
-reportPropertyTypeMismatch = "error"
-reportUninitializedInstanceVariable = "error"
-reportUnnecessaryTypeIgnoreComment = "error"
+[tool.ty.environment]
+python-version = "3.12"
-# This is a big TODO list of current typing problems
-# Pick an item off the list, change to "error", and send a pull request to fix it!
-# You can find more information on a rule at https://github.com/microsoft/pyright/blob/main/docs/configuration.md
-reportConstantRedefinition = "hint" # Requires fixing nonstandard variable conventions
-reportMissingSuperCall = "hint" # Requires adding missing super().__init__() calls
-reportMissingTypeStubs = "hint" # Can be fixed by switching 'ics' to alternative such as 'icalendar'
-reportPrivateUsage = "hint" # Can be fixed by using TLS properly (Ruff S323)
-reportUnnecessaryComparison = "hint" # Requires cleaning up some dead code
-reportUntypedNamedTuple = "hint" # Requires moving to different type of named tuple
-reportUnusedCallResult = "none" # Forces you to assign unused retvals to '_', very noisy.
+[tool.ty.rules]
+division-by-zero = "warn"
+missing-override-decorator = "warn"
+possibly-missing-attribute = "warn"
+possibly-missing-import = "warn"
+possibly-unresolved-reference = "warn"
+unsupported-dynamic-base = "warn"
+
+[tool.ty.terminal]
+error-on-warning = true
[tool.ruff]
target-version = "py312"
@@ -37,11 +27,6 @@ ignore = [
# You can find more information on a rule at https://docs.astral.sh/ruff/rules/
"A001", # variable name is shadowing a python builtin
"A002", # function argument name is shadowing a python builtin
- "ANN001", # missing type annotation for function argument
- "ANN201", # missing return annotation for public function
- "ANN202", # missing return annotation for private function
- "ANN204", # missing return annotation for special method
- "ANN206", # missing return annotation for classmethod
"ANN401", # use of Any as function parameter
"ARG005", # unused lambda argument
"B006", # do not use mutable datastructures for arg defaults
diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py
index 6fc70bbae7e..37de9a936aa 100755
--- a/dev-tools/scripts/releaseWizard.py
+++ b/dev-tools/scripts/releaseWizard.py
@@ -67,7 +67,7 @@ editor = None
# Edit this to add other global jinja2 variables or filters
-def expand_jinja(text: str, vars: dict[str, Any] | None = None):
+def expand_jinja(text: str, vars: dict[str, Any] | None = None) -> str:
global_vars: dict[str, Any] = OrderedDict(
{
"script_version": state.script_version,
@@ -135,7 +135,7 @@ def expand_jinja(text: str, vars: dict[str, Any] | None = None):
return filled
-def replace_templates(text: str):
+def replace_templates(text: str) -> str:
tpl_lines: list[str] = []
for line in text.splitlines():
if match := re.search(r"^\(\( template=(.+?) \)\)", line):
@@ -146,11 +146,11 @@ def replace_templates(text: str):
return "\n".join(tpl_lines)
-def getScriptVersion():
+def getScriptVersion() -> str:
return scriptutil.find_current_version()
-def get_editor():
+def get_editor() -> str:
global editor
if editor is None:
if "EDITOR" in os.environ:
@@ -166,7 +166,7 @@ def get_editor():
return editor
-def check_prerequisites(todo: Any = None):
+def check_prerequisites(todo: "Todo | None" = None) -> bool:
if sys.version_info < (3, 4):
sys.exit("Script requires Python v3.4 or later")
try:
@@ -201,11 +201,11 @@ def check_prerequisites(todo: Any = None):
epoch = datetime.fromtimestamp(timestamp=0, tz=UTC)
-def unix_time_millis(dt: datetime):
+def unix_time_millis(dt: datetime) -> int:
return int((dt - epoch).total_seconds() * 1000.0)
-def bootstrap_todos(todo_list: list[Any]):
+def bootstrap_todos(todo_list: list[Any]) -> list[Any]:
# Establish links from commands to to_do for finding todo vars
for tg in todo_list:
if dry_run:
@@ -227,7 +227,7 @@ def bootstrap_todos(todo_list: list[Any]):
return todo_list
-def maybe_remove_rc_from_svn():
+def maybe_remove_rc_from_svn() -> None:
todo = state.get_todo_by_id("import_svn")
if todo and todo.is_done():
print("import_svn done")
@@ -253,7 +253,7 @@ class SecretYamlObject(yaml.YAMLObject):
@classmethod
@override
- def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any):
+ def to_yaml(cls: type[Self], dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:
print("Dumping object %s" % type(data))
new_data = copy.deepcopy(data)
@@ -263,17 +263,17 @@ class SecretYamlObject(yaml.YAMLObject):
for item in data.__dict__:
if item in new_data.__dict__ and new_data.__dict__[item] is None:
del new_data.__dict__[item]
- return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
+ return dumper.represent_yaml_object(cls.yaml_tag, new_data, cls, flow_style=cls.yaml_flow_style)
-def str_presenter(dumper: yaml.Dumper, data: str):
+def str_presenter(dumper: yaml.Dumper, data: str) -> yaml.ScalarNode:
if len(data.split("\n")) > 1: # check for multiline string
- return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") # TODO: fix me # pyright: ignore[reportUnknownMemberType]
- return dumper.represent_scalar("tag:yaml.org,2002:str", data) # TODO: fix me # pyright: ignore[reportUnknownMemberType]
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data)
class ReleaseState:
- def __init__(self, config_path: str, release_version: str, script_version: str):
+ def __init__(self, config_path: str, release_version: str, script_version: str) -> None:
self.script_version = script_version
self.config_path = config_path
self.todo_groups: list[TodoGroup] = []
@@ -291,7 +291,7 @@ class ReleaseState:
self.script_branch_type = "feature"
self.set_release_version(release_version)
- def set_release_version(self, version: str):
+ def set_release_version(self, version: str) -> None:
self.validate_release_version(self.script_branch_type, self.script_branch, version)
self.release_version = version
v = Version.parse(version)
@@ -306,32 +306,32 @@ class ReleaseState:
else:
self.release_type = "bugfix"
- def is_released(self):
+ def is_released(self) -> bool:
todo = self.get_todo_by_id("announce_lucene")
assert todo
return todo.is_done()
- def get_gpg_key(self):
+ def get_gpg_key(self) -> str | None:
gpg_task = self.get_todo_by_id("gpg")
assert gpg_task
if gpg_task.is_done():
return gpg_task.get_state()["gpg_key"]
return None
- def get_release_date(self):
+ def get_release_date(self) -> datetime | None:
publish_task = self.get_todo_by_id("publish_maven")
assert publish_task
if publish_task.is_done():
return unix_to_datetime(publish_task.get_state()["done_date"])
return None
- def get_release_date_iso(self):
+ def get_release_date_iso(self) -> str:
release_date = self.get_release_date()
if release_date is None:
return "yyyy-mm-dd"
return release_date.isoformat()[:10]
- def get_latest_version(self):
+ def get_latest_version(self) -> str:
if self.latest_version is None:
versions = self.get_mirrored_versions()
latest = versions[0]
@@ -343,14 +343,14 @@ class ReleaseState:
assert state.latest_version
return state.latest_version
- def get_mirrored_versions(self):
+ def get_mirrored_versions(self) -> list[str]:
if state.mirrored_versions is None:
releases_str = load("https://projects.apache.org/json/foundation/releases.json", "utf-8")
releases: dict[str, str] = json.loads(releases_str)["lucene"]
state.mirrored_versions = [r for r in list(map(lambda y: y[7:], filter(lambda x: x.startswith("lucene-"), list(releases.keys()))))]
return state.mirrored_versions
- def get_mirrored_versions_to_delete(self):
+ def get_mirrored_versions_to_delete(self) -> list[str]:
versions = self.get_mirrored_versions()
to_keep = versions
if state.release_type == "major":
@@ -366,11 +366,11 @@ class ReleaseState:
raise Exception("Release version %s must have same major version as current minor or lts release")
return [ver for ver in versions if ver not in to_keep]
- def get_main_version(self):
+ def get_main_version(self) -> str:
v = Version.parse(self.get_latest_version())
return "%s.%s.%s" % (v.major + 1, 0, 0)
- def get_latest_lts_version(self):
+ def get_latest_lts_version(self) -> str:
versions = self.get_mirrored_versions()
latest = self.get_latest_version()
lts_prefix = "%s." % (Version.parse(latest).major - 1)
@@ -381,7 +381,7 @@ class ReleaseState:
latest_lts = ver
return latest_lts
- def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str):
+ def validate_release_version(self, branch_type: BranchType | str, branch: str, release_version: str) -> None:
ver = Version.parse(release_version)
# print("release_version=%s, ver=%s" % (release_version, ver))
if branch_type == BranchType.release:
@@ -402,7 +402,7 @@ class ReleaseState:
if not getScriptVersion() == release_version:
print("WARNING: Expected release version %s when on branch %s, but got %s" % (getScriptVersion(), branch, release_version))
- def get_base_branch_name(self):
+ def get_base_branch_name(self) -> str:
v = Version.parse(self.release_version)
if v.is_major_release():
return "main"
@@ -412,7 +412,7 @@ class ReleaseState:
return self.get_minor_branch_name()
return self.release_branch
- def clear_rc(self):
+ def clear_rc(self) -> None:
if ask_yes_no("Are you sure? This will clear and restart RC%s" % self.rc_number):
maybe_remove_rc_from_svn()
for g in list(filter(lambda x: x.in_rc_loop(), self.todo_groups)):
@@ -426,7 +426,7 @@ class ReleaseState:
print("WARN: Failed to clear %s, please do it manually with higher privileges" % self.get_rc_folder())
self.save()
- def new_rc(self):
+ def new_rc(self) -> None:
if ask_yes_no("Are you sure? This will abort current RC"):
maybe_remove_rc_from_svn()
dict = {}
@@ -439,7 +439,7 @@ class ReleaseState:
self.rc_number += 1
self.save()
- def to_dict(self):
+ def to_dict(self) -> dict[str, Any]:
tmp_todos = {}
for todo_id in self.todos:
t = self.todos[todo_id]
@@ -457,7 +457,7 @@ class ReleaseState:
dictionary["latest_version"] = self.latest_version
return dictionary
- def restore_from_dict(self, dictionary: dict[str, Any]):
+ def restore_from_dict(self, dictionary: dict[str, Any]) -> None:
self.script_version = dictionary["script_version"]
assert dictionary["release_version"] == self.release_version
if "start_date" in dictionary:
@@ -474,7 +474,7 @@ class ReleaseState:
else:
print("Warning: Could not restore state for %s, Todo definition not found" % todo_id)
- def load(self):
+ def load(self) -> None:
if os.path.exists(os.path.join(self.config_path, self.release_version, "state.yaml")):
state_file = os.path.join(self.config_path, self.release_version, "state.yaml")
with open(state_file) as fp:
@@ -485,7 +485,7 @@ class ReleaseState:
except Exception as e:
print("Failed to load state from %s: %s" % (state_file, e))
- def save(self):
+ def save(self) -> None:
print("Saving")
if not os.path.exists(os.path.join(self.config_path, self.release_version)):
print("Creating folder %s" % os.path.join(self.config_path, self.release_version))
@@ -494,7 +494,7 @@ class ReleaseState:
with open(os.path.join(self.config_path, self.release_version, "state.yaml"), "w") as fp:
yaml.dump(self.to_dict(), fp, sort_keys=False, default_flow_style=False)
- def clear(self):
+ def clear(self) -> None:
self.previous_rcs = {}
self.rc_number = 1
for t_id in self.todos:
@@ -502,72 +502,72 @@ class ReleaseState:
t.state = {}
self.save()
- def get_rc_number(self):
+ def get_rc_number(self) -> int:
return self.rc_number
- def get_current_git_rev(self):
+ def get_current_git_rev(self) -> str:
try:
return run("git rev-parse HEAD", cwd=self.get_git_checkout_folder()).strip()
except Exception:
return "<git-rev>"
- def get_group_by_id(self, id: str):
+ def get_group_by_id(self, id: str) -> "TodoGroup | None":
lst: list[TodoGroup] = list(filter(lambda x: x.id == id, self.todo_groups))
if len(lst) == 1:
return lst[0]
return None
- def get_todo_by_id(self, id: str):
+ def get_todo_by_id(self, id: str) -> "Todo | None":
lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0]
return None
- def get_todo_state_by_id(self, id: str):
+ def get_todo_state_by_id(self, id: str) -> dict[Any, Any]:
lst: list[Todo] = list(filter(lambda x: x.id == id, self.todos.values()))
if len(lst) == 1:
return lst[0].state
- return cast("dict[Any, Any]", {})
+ return {}
- def get_release_folder(self):
+ def get_release_folder(self) -> str:
folder = os.path.join(self.config_path, self.release_version)
if not os.path.exists(folder):
print("Creating folder %s" % folder)
Path(folder).mkdir(parents=True)
return folder
- def get_rc_folder(self):
+ def get_rc_folder(self) -> str:
folder = os.path.join(self.get_release_folder(), "RC%d" % self.rc_number)
if not os.path.exists(folder):
print("Creating folder %s" % folder)
Path(folder).mkdir(parents=True)
return folder
- def get_dist_folder(self):
+ def get_dist_folder(self) -> str:
folder = os.path.join(self.get_rc_folder(), "dist")
return folder
- def get_git_checkout_folder(self):
+ def get_git_checkout_folder(self) -> str:
folder = os.path.join(self.get_release_folder(), "lucene")
return folder
- def get_website_git_folder(self):
+ def get_website_git_folder(self) -> str:
folder = os.path.join(self.get_release_folder(), "lucene-site")
return folder
- def get_minor_branch_name(self):
+ def get_minor_branch_name(self) -> str:
latest = state.get_latest_version()
v = Version.parse(latest)
return "branch_%s_%s" % (v.major, v.minor)
- def get_stable_branch_name(self):
+ def get_stable_branch_name(self) -> str:
if self.release_type == "major":
v = Version.parse(self.get_main_version())
else:
v = Version.parse(self.get_latest_version())
return "branch_%sx" % v.major
- def get_next_version(self):
+ def get_next_version(self) -> str | None:
if self.release_type == "major":
return "%s.0.0" % (self.release_version_major + 1)
if self.release_type == "minor":
@@ -576,10 +576,10 @@ class ReleaseState:
return "%s.%s.%s" % (self.release_version_major, self.release_version_minor, self.release_version_bugfix + 1)
return None
- def get_java_home(self):
+ def get_java_home(self) -> str:
return self.get_java_home_for_version(self.release_version)
- def get_java_home_for_version(self, version: str):
+ def get_java_home_for_version(self, version: str) -> str:
v = Version.parse(version)
java_ver = java_versions[v.major]
java_home_var = "JAVA%s_HOME" % java_ver
@@ -587,21 +587,21 @@ class ReleaseState:
return os.environ[java_home_var]
raise Exception("Script needs environment variable %s" % java_home_var)
- def get_java_cmd_for_version(self, version: str):
+ def get_java_cmd_for_version(self, version: str) -> str:
return os.path.join(self.get_java_home_for_version(version), "bin", "java")
- def get_java_cmd(self):
+ def get_java_cmd(self) -> str:
return os.path.join(self.get_java_home(), "bin", "java")
- def get_todo_states(self):
- states: dict[str, dict[Any, Any]] = {}
+ def get_todo_states(self) -> dict[str, dict[Any, Any]]:
+ states = {}
if self.todos:
for todo_id in self.todos:
t = self.todos[todo_id]
states[todo_id] = copy.deepcopy(t.state)
return states
- def init_todos(self, groups: list[Any]):
+ def init_todos(self, groups: list[Any]) -> None:
self.todo_groups = groups
self.todos = {}
for g in self.todo_groups:
@@ -613,7 +613,7 @@ class TodoGroup(SecretYamlObject):
yaml_tag = "!TodoGroup"
hidden_fields = []
- def __init__(self, id: str, title: str, description: str, todos: list[Any], is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None):
+ def __init__(self, id: str, title: str, description: str, todos: "list[Todo]", is_in_rc_loop: bool | None = None, depends: str | list[str] | None = None) -> None:
self.id = id
self.title = title
self.description = description
@@ -623,30 +623,30 @@ class TodoGroup(SecretYamlObject):
@classmethod
@override
- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "TodoGroup":
fields = loader.construct_mapping(node, deep=True)
return TodoGroup(**cast("dict[str, Any]", fields))
- def num_done(self):
+ def num_done(self) -> int:
return sum(1 for x in self.todos if x.is_done() > 0)
- def num_applies(self):
+ def num_applies(self) -> int:
count = sum(1 for x in self.todos if x.applies(state.release_type))
# print("num_applies=%s" % count)
return count
- def is_done(self):
+ def is_done(self) -> bool:
# print("Done=%s, applies=%s" % (self.num_done(), self.num_applies()))
return self.num_done() >= self.num_applies()
- def get_title(self):
+ def get_title(self) -> str:
# print("get_title: %s" % self.is_done())
prefix = ""
if self.is_done():
prefix = "✓ "
return "%s%s (%d/%d)" % (prefix, self.title, self.num_done(), self.num_applies())
- def get_submenu(self):
+ def get_submenu(self) -> ConsoleMenu:
menu = ConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(), clear_screen=False)
menu.exit_item = CustomExitItem("Return")
for todo in self.get_todos():
@@ -654,17 +654,17 @@ class TodoGroup(SecretYamlObject):
menu.append_item(todo.get_menu_item())
return menu
- def get_menu_item(self):
+ def get_menu_item(self) -> SubmenuItem:
item = SubmenuItem(self.get_title, self.get_submenu())
return item
- def get_todos(self):
+ def get_todos(self) -> "list[Todo]":
return self.todos
- def in_rc_loop(self):
+ def in_rc_loop(self) -> bool:
return self.is_in_rc_loop is True
- def get_description(self):
+ def get_description(self) -> str | None:
desc = self.description
if desc:
return expand_jinja(desc)
@@ -696,14 +696,14 @@ class Todo(SecretYamlObject):
done: bool | None = None,
types: list[str] | str | None = None,
links: list[str] | None = None,
- commands: Any = None,
- user_input: Any = None,
+ commands: "Commands | None" = None,
+ user_input: "UserInput | None" = None,
depends: str | list[str] | None = None,
vars: dict[str, Any] | None = None,
asciidoc: str | None = None,
persist_vars: dict[str, str] | None = None,
function: str | None = None,
- ):
+ ) -> None:
self.id = id
self.title = title
self.description = description
@@ -732,11 +732,11 @@ class Todo(SecretYamlObject):
@classmethod
@override
- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Todo":
fields = loader.construct_mapping(node, deep=True)
return Todo(**cast("dict[str, Any]", fields))
- def get_vars(self):
+ def get_vars(self) -> dict[str, str]:
myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
@@ -747,7 +747,7 @@ class Todo(SecretYamlObject):
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def set_done(self, is_done: bool | None):
+ def set_done(self, is_done: bool | None) -> None:
if is_done:
self.state["done_date"] = unix_time_millis(datetime.now(tz=UTC))
if self.persist_vars:
@@ -757,21 +757,21 @@ class Todo(SecretYamlObject):
self.state.clear()
self.state["done"] = is_done
- def applies(self, type: str):
+ def applies(self, type: str) -> bool:
if self.types:
return type in self.types
return True
- def is_done(self):
+ def is_done(self) -> bool:
return "done" in self.state and self.state["done"] is True
- def get_title(self):
+ def get_title(self) -> str:
prefix = ""
if self.is_done():
prefix = "✓ "
return expand_jinja("%s%s" % (prefix, self.title), self.get_vars_and_state())
- def display_and_confirm(self):
+ def display_and_confirm(self) -> None:
try:
if self.depends:
for dep in ensure_list(self.depends):
@@ -827,48 +827,48 @@ class Todo(SecretYamlObject):
except Exception as e:
print("ERROR while executing todo %s (%s)" % (self.get_title(), e))
- def get_menu_item(self):
+ def get_menu_item(self) -> FunctionItem:
return FunctionItem(self.get_title(), self.display_and_confirm)
- def clone(self):
+ def clone(self) -> "Todo":
clone = Todo(self.id, self.title, description=self.description)
clone.state = copy.deepcopy(self.state)
return clone
- def clear(self):
+ def clear(self) -> None:
self.state.clear()
self.set_done(False)
- def get_state(self):
+ def get_state(self) -> dict[Any, Any]:
return self.state
- def get_description(self):
+ def get_description(self) -> str | None:
desc = self.description
if desc:
return expand_jinja(desc, vars=self.get_vars_and_state())
return None
- def get_post_description(self):
+ def get_post_description(self) -> str | None:
if self.post_description:
return expand_jinja(self.post_description, vars=self.get_vars_and_state())
return None
- def get_commands(self):
+ def get_commands(self) -> "Commands | None":
cmds = self.commands
return cmds
- def get_asciidoc(self):
+ def get_asciidoc(self) -> str | None:
if self.asciidoc:
return expand_jinja(self.asciidoc, vars=self.get_vars_and_state())
return None
- def get_vars_and_state(self):
+ def get_vars_and_state(self) -> dict[str, str]:
d = self.get_vars().copy()
d.update(self.get_state())
return d
-def get_release_version():
+def get_release_version() -> str:
v = str(input("Which version are you releasing? (x.y.z) "))
try:
version = Version.parse(v)
@@ -879,30 +879,33 @@ def get_release_version():
return str(version)
-def get_subtitle():
+def get_subtitle() -> str:
applying_groups = list(filter(lambda x: x.num_applies() > 0, state.todo_groups))
done_groups = sum(1 for x in applying_groups if x.is_done())
return "Please complete the below checklist (Complete: %s/%s)" % (done_groups, len(applying_groups))
-def get_todo_menuitem_title():
+def get_todo_menuitem_title() -> str:
return "Go to checklist (RC%d)" % (state.rc_number)
-def get_releasing_text():
+def get_releasing_text() -> str:
return "Releasing Lucene %s RC%d" % (state.release_version, state.rc_number)
-def get_start_new_rc_menu_title():
+def get_start_new_rc_menu_title() -> str:
return "Abort RC%d and start a new RC%d" % (state.rc_number, state.rc_number + 1)
-def start_new_rc():
+def start_new_rc() -> None:
state.new_rc()
print("Started RC%d" % state.rc_number)
-def reset_state():
+state: ReleaseState
+
+
+def reset_state() -> None:
global state
if ask_yes_no("Are you sure? This will erase all current progress"):
maybe_remove_rc_from_svn()
@@ -910,16 +913,16 @@ def reset_state():
state.clear()
-def template(name: str, vars: dict[str, Any] | None = None):
+def template(name: str, vars: dict[str, Any] | None = None) -> str:
return expand_jinja(templates[name], vars=vars)
-def help():
+def help() -> None:
print(template("help"))
pause()
-def ensure_list(o: Any):
+def ensure_list(o: Any) -> list[Any]:
if o is None:
return cast("list[Any]", [])
if not isinstance(o, list):
@@ -927,7 +930,7 @@ def ensure_list(o: Any):
return cast("list[Any]", o)
-def open_file(filename: str):
+def open_file(filename: str) -> None:
print("Opening file %s" % filename)
if platform.system().startswith("Win"):
run("start %s" % filename)
@@ -935,15 +938,15 @@ def open_file(filename: str):
run("open %s" % filename)
-def expand_multiline(cmd_txt: str, indent: int = 0):
+def expand_multiline(cmd_txt: str, indent: int = 0) -> str:
return re.sub(r" +", " %s\n %s" % (Commands.cmd_continuation_char, " " * indent), cmd_txt)
-def unix_to_datetime(unix_stamp: int):
+def unix_to_datetime(unix_stamp: int) -> datetime:
return datetime.fromtimestamp(timestamp=unix_stamp / 1000, tz=UTC)
-def generate_asciidoc():
+def generate_asciidoc() -> None:
base_filename = os.path.join(state.get_release_folder(), "lucene_release_%s" % (state.release_version.replace(r"\.", "_")))
filename_adoc = "%s.adoc" % base_filename
@@ -1025,7 +1028,7 @@ def generate_asciidoc():
pause()
-def load_rc():
+def load_rc() -> dict[str, Any]:
lucenerc = os.path.expanduser("~/.lucenerc")
try:
with open(lucenerc) as fp:
@@ -1034,7 +1037,7 @@ def load_rc():
return cast("dict[str, Any]", {})
-def store_rc(release_root: str, release_version: str | None = None):
+def store_rc(release_root: str, release_version: str | None = None) -> None:
lucenerc = os.path.expanduser("~/.lucenerc")
dict = {}
dict["root"] = release_root
@@ -1044,7 +1047,7 @@ def store_rc(release_root: str, release_version: str | None = None):
json.dump(dict, fp, indent=2)
-def release_other_version():
+def release_other_version() -> None:
if not state.is_released():
maybe_remove_rc_from_svn()
store_rc(state.config_path, None)
@@ -1052,28 +1055,28 @@ def release_other_version():
sys.exit(0)
-def file_to_string(filename: str):
+def file_to_string(filename: str) -> str:
with open(filename, encoding="utf8") as f:
return f.read().strip()
-def download_keys():
+def download_keys() -> None:
download("KEYS", "https://archive.apache.org/dist/lucene/KEYS", state.config_path)
-def keys_downloaded():
+def keys_downloaded() -> bool:
return os.path.exists(os.path.join(state.config_path, "KEYS"))
-def dump_yaml():
+def dump_yaml() -> None:
file = open(os.path.join(script_path, "releaseWizard.yaml"), "w")
yaml.add_representer(str, str_presenter)
- yaml.Dumper.ignore_aliases = lambda self, data: True # TODO: fix me # pyright: ignore[reportUnknownLambdaType]
+ yaml.Dumper.ignore_aliases = lambda self, data: True # ty:ignore[invalid-assignment]
dump_obj: dict[str, Any] = {"templates": templates, "groups": state.todo_groups}
yaml.dump(dump_obj, width=180, stream=file, sort_keys=False, default_flow_style=False)
-def parse_config():
+def parse_config() -> argparse.Namespace:
description = "Script to guide a RM through the whole release process"
parser = argparse.ArgumentParser(description=description, epilog="Go push that release!", formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--dry-run", dest="dry", action="store_true", default=False, help="Do not execute any commands, but echo them instead. Display extra debug info")
@@ -1083,7 +1086,7 @@ def parse_config():
return config
-def load(urlString: str, encoding: str = "utf-8"):
+def load(urlString: str, encoding: str = "utf-8") -> str:
try:
content = urllib.request.urlopen(urlString).read().decode(encoding)
except Exception as e:
@@ -1092,7 +1095,7 @@ def load(urlString: str, encoding: str = "utf-8"):
return content
-def configure_pgp(gpg_todo: Todo):
+def configure_pgp(gpg_todo: Todo) -> bool:
print("Based on your Apache ID we'll lookup your key online\nand through this complete the 'gpg' prerequisite task.\n")
gpg_state = gpg_todo.get_state()
id = str(input("Please enter your Apache id: (ENTER=skip) "))
@@ -1237,7 +1240,7 @@ def configure_pgp(gpg_todo: Todo):
return True
-def pause(fun: Callable[[], None] | None = None):
+def pause(fun: Callable[[], None] | None = None) -> None:
if fun:
fun()
input("\nPress ENTER to continue...")
@@ -1245,15 +1248,19 @@ def pause(fun: Callable[[], None] | None = None):
class CustomExitItem(ExitItem):
@override
- def show(self, index: int, available_width: int | None = None):
+ def show(self, index: int, available_width: int | None = None) -> Any:
return super(CustomExitItem, self).show(index)
@override
- def get_return(self):
+ def get_return(self) -> str:
return ""
-def main():
+templates: Any
+lucene_news_file: str
+
+
+def main() -> None:
global state
global dry_run
global templates
@@ -1359,11 +1366,11 @@ script_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(script_path)
-def git_checkout_folder():
+def git_checkout_folder() -> str:
return state.get_git_checkout_folder()
-def tail_file(file: str, lines: int):
+def tail_file(file: str, lines: int) -> None:
bufsize = 8192
fsize = os.stat(file).st_size
with open(file) as f:
@@ -1383,7 +1390,9 @@ def tail_file(file: str, lines: int):
break
-def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
+def run_with_log_tail(
+ command: str | list[str], cwd: str | None, logfile: str | None = None, tail_lines: int = 10, tee: bool | None = False, live: bool | None = False, shell: bool | None = None
+) -> int | None:
fh = sys.stdout
if logfile:
logdir = os.path.dirname(logfile)
@@ -1398,7 +1407,7 @@ def run_with_log_tail(command: str | list[str], cwd: str | None, logfile: str |
return rc
-def ask_yes_no(text: str):
+def ask_yes_no(text: str) -> bool:
answer = None
while answer not in ["y", "n"]:
answer = str(input("\nQ: %s (y/n): " % text))
@@ -1406,16 +1415,16 @@ def ask_yes_no(text: str):
return answer == "y"
-def abbreviate_line(line: str, width: int):
+def abbreviate_line(line: str, width: int) -> str:
line = line.rstrip()
if len(line) > width:
- line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :])
+ line = "%s.....%s" % (line[: (width / 2 - 5)], line[-(width / 2) :]) # ty:ignore[invalid-argument-type]
else:
line = "%s%s" % (line, " " * (width - len(line) + 2))
return line
-def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False):
+def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None = False) -> None:
if not tee:
if not stdout:
print("[line %s] %s" % (linenum, abbreviate_line(line, 80)), end="\r")
@@ -1425,7 +1434,7 @@ def print_line_cr(line: str, linenum: int, stdout: bool = True, tee: bool | None
print(line.rstrip())
-def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None):
+def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sys.stdout, tee: bool | None = False, live: bool | None = False, shell: bool | None = None) -> int | None:
doShell = "&&" in command or "&" in command or shell is not None
if not doShell and not isinstance(command, list):
command = shlex.split(command)
@@ -1502,15 +1511,15 @@ def run_follow(command: str | list[str], cwd: str | None = None, fh: TextIO = sy
return rc
-def is_windows():
+def is_windows() -> bool:
return platform.system().startswith("Win")
-def is_mac():
+def is_mac() -> bool:
return platform.system().startswith("Darwin")
-def is_linux():
+def is_linux() -> bool:
return platform.system().startswith("Linux")
@@ -1532,7 +1541,7 @@ class Commands(SecretYamlObject):
vars: dict[str, Any] | None = None,
todo_id: str | None = None,
remove_files: list[str] | None = None,
- ):
+ ) -> None:
self.root_folder = root_folder
self.commands_text = commands_text
self.vars = vars
@@ -1549,11 +1558,11 @@ class Commands(SecretYamlObject):
@classmethod
@override
- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Commands":
fields = loader.construct_mapping(node, deep=True)
return Commands(**cast("dict[str, Any]", fields))
- def run(self): # pylint: disable=inconsistent-return-statements # TODO
+ def run(self) -> bool | None:
root = self.get_root_folder()
if self.commands_text:
@@ -1667,19 +1676,19 @@ class Commands(SecretYamlObject):
print("WARNING: One or more commands failed, you may want to check the logs")
return success
- def get_root_folder(self):
+ def get_root_folder(self) -> str:
return cast("str", self.jinjaify(self.root_folder))
- def get_commands_text(self):
+ def get_commands_text(self) -> str | list[str] | None:
return self.jinjaify(self.commands_text)
- def get_run_text(self):
+ def get_run_text(self) -> str | list[str] | None:
return self.jinjaify(self.run_text)
- def get_remove_files(self):
+ def get_remove_files(self) -> str | list[str] | None:
return self.jinjaify(self.remove_files)
- def get_vars(self):
+ def get_vars(self) -> dict[str, str]:
myvars: dict[str, str] = {}
if self.vars:
for k in self.vars:
@@ -1690,7 +1699,7 @@ class Commands(SecretYamlObject):
myvars[k] = expand_jinja(val, vars=myvars)
return myvars
- def jinjaify(self, data: str | list[str] | None, join: bool = False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False) -> str | list[str] | None:
if not data:
return None
v = self.get_vars()
@@ -1708,7 +1717,7 @@ class Commands(SecretYamlObject):
return expand_jinja(data, v)
-def abbreviate_homedir(line: str):
+def abbreviate_homedir(line: str) -> str:
if is_windows():
if "HOME" in os.environ:
return re.sub(r"([^/]|\b)%s" % os.path.expanduser("~"), "\\1%HOME%", line)
@@ -1737,7 +1746,7 @@ class Command(SecretYamlObject):
redirect: str | None = None,
redirect_append: bool | None = None,
shell: bool | None = None,
- ):
+ ) -> None:
self.cmd = cmd
self.cwd = cwd
self.comment = comment
@@ -1766,21 +1775,21 @@ class Command(SecretYamlObject):
@classmethod
@override
- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "Command":
fields = loader.construct_mapping(node, deep=True)
return Command(**cast("dict[str, Any]", fields))
- def get_comment(self):
+ def get_comment(self) -> str:
return str(self.jinjaify(self.comment))
- def get_redirect(self):
+ def get_redirect(self) -> str:
return str(self.jinjaify(self.redirect))
- def get_cmd(self):
+ def get_cmd(self) -> str:
return str(self.jinjaify(self.cmd, join=True))
- def get_vars(self):
- myvars: dict[str, str] = {}
+ def get_vars(self) -> dict[str, str]:
+ myvars = {}
if self.vars:
for k in self.vars:
val = self.vars[k]
@@ -1791,10 +1800,10 @@ class Command(SecretYamlObject):
return myvars
@override
- def __str__(self):
+ def __str__(self) -> str:
return self.get_cmd()
- def jinjaify(self, data: str | list[str] | None, join: bool = False):
+ def jinjaify(self, data: str | list[str] | None, join: bool = False) -> str | list[str] | None:
if data is None:
return None
v = self.get_vars()
@@ -1811,7 +1820,7 @@ class Command(SecretYamlObject):
return res
return expand_jinja(data, v)
- def display_cmd(self):
+ def display_cmd(self) -> list[str]:
lines: list[str] = []
if self.comment:
if is_windows():
@@ -1833,18 +1842,18 @@ class Command(SecretYamlObject):
class UserInput(SecretYamlObject):
yaml_tag = "!UserInput"
- def __init__(self, name: str, prompt: str, type: str | None = None):
+ def __init__(self, name: str, prompt: str, type: str | None = None) -> None:
self.type = type
self.prompt = prompt
self.name = name
@classmethod
@override
- def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode):
+ def from_yaml(cls, loader: yaml.Loader, node: yaml.MappingNode) -> "UserInput":
fields = loader.construct_mapping(node, deep=True)
return UserInput(**cast("dict[str, Any]", fields))
- def run(self, dict: dict[str, Any] | None = None):
+ def run(self, dict: dict[str, Any] | None = None) -> str | int | None:
correct = False
while not correct:
try:
@@ -1860,7 +1869,7 @@ class UserInput(SecretYamlObject):
return result
-def create_ical(_todo: Todo): # pylint: disable=unused-argument
+def create_ical(_todo: Todo) -> bool:
if ask_yes_no("Do you want to add a Calendar reminder for the close vote time?"):
c = Calendar()
e = Event()
@@ -1870,7 +1879,7 @@ def create_ical(_todo: Todo): # pylint: disable=unused-argument
c.events.add(e)
ics_file = os.path.join(state.get_rc_folder(), "vote_end.ics")
with open(ics_file, "w") as my_file:
- my_file.writelines(c) # pyright: ignore[reportArgumentType]
+ my_file.writelines(iter(c))
open_file(ics_file)
return True
@@ -1890,12 +1899,12 @@ non_working = (
)
-def vote_close_72h_date():
+def vote_close_72h_date() -> datetime:
# Voting open at least 72 hours according to ASF policy
return datetime.now(tz=UTC) + timedelta(hours=73)
-def vote_close_72h_holidays():
+def vote_close_72h_holidays() -> list[str] | None:
days = 0
day_offset = -1
holidays: list[str] = []
@@ -1913,7 +1922,7 @@ def vote_close_72h_holidays():
return holidays if len(holidays) > 0 else None
-def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argument
+def prepare_announce_lucene(_todo: Todo) -> bool:
if not os.path.exists(lucene_news_file):
lucene_text = expand_jinja("(( template=announce_lucene ))")
with open(lucene_news_file, "w") as fp:
@@ -1924,7 +1933,7 @@ def prepare_announce_lucene(_todo: Todo): # pylint: disable=unused-argument
return True
-def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argument
+def check_artifacts_available(_todo: Todo) -> bool:
cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
try:
load(cdnUrl)
@@ -1944,12 +1953,12 @@ def check_artifacts_available(_todo: Todo): # pylint: disable=unused-argument
return True
-def set_java_home(version: str):
+def set_java_home(version: str) -> None:
os.environ["JAVA_HOME"] = state.get_java_home_for_version(version)
os.environ["JAVACMD"] = state.get_java_cmd_for_version(version)
-def load_lines(file: str, from_line: int = 0):
+def load_lines(file: str, from_line: int = 0) -> list[str]:
if os.path.exists(file):
with open(file) as fp:
return fp.readlines()[from_line:]
diff --git a/dev-tools/scripts/releasedJirasRegex.py b/dev-tools/scripts/releasedJirasRegex.py
index 0dd5ccd6360..f972a8d2cb7 100755
--- a/dev-tools/scripts/releasedJirasRegex.py
+++ b/dev-tools/scripts/releasedJirasRegex.py
@@ -32,7 +32,7 @@ from scriptutil import Version
# does not find Bugzilla bugs or JIRAs not mentioned at the beginning of
# bullets or numbered entries.
#
-def print_released_jiras_regex(version: str, filename: str):
+def print_released_jiras_regex(version: str, filename: str) -> None:
release_boundary_re = re.compile(r"\s*====*\s+(.*)\s+===")
version_re = re.compile(r"%s(?:$|[^-])" % version)
bullet_re = re.compile(r"\s*(?:[-*]|\d+\.(?=(?:\s|(?:LUCENE)-)))(.*)")
@@ -74,14 +74,14 @@ def print_released_jiras_regex(version: str, filename: str):
print()
-def read_config():
+def read_config() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Prints a regex matching JIRAs fixed in the given version by parsing the given CHANGES.txt file")
parser.add_argument("version", type=Version.parse, help="Version of the form X.Y.Z")
parser.add_argument("changes", help="CHANGES.txt file to parse")
return parser.parse_args()
-def main():
+def main() -> None:
config = read_config()
print_released_jiras_regex(config.version, config.changes)
diff --git a/dev-tools/scripts/reproduceJenkinsFailures.py b/dev-tools/scripts/reproduceJenkinsFailures.py
index 46ddbfb1600..a889f51c16e 100644
--- a/dev-tools/scripts/reproduceJenkinsFailures.py
+++ b/dev-tools/scripts/reproduceJenkinsFailures.py
@@ -72,7 +72,7 @@ description = dedent("""\
defaultIters = 5
-def readConfig():
+def readConfig() -> argparse.Namespace:
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=description)
parser.add_argument("url", metavar="URL", help="Points to the Jenkins log to parse")
parser.add_argument("--no-git", dest="useGit", action="store_false", default=True, help='Do not run "git" at all')
@@ -80,7 +80,7 @@ def readConfig():
return parser.parse_args()
-def runOutput(cmd: str):
+def runOutput(cmd: str) -> str:
print("[repro] %s" % cmd)
try:
return subprocess.check_output(cmd.split(" "), universal_newlines=True).strip()
@@ -89,16 +89,21 @@ def runOutput(cmd: str):
# Remembers non-zero exit code in lastFailureCode unless rememberFailure==False
-def run(cmd: str, rememberFailure: bool = True):
+def run(cmd: str, rememberFailure: bool = True) -> int:
global lastFailureCode
print("[repro] %s" % cmd)
- code = os.system(cmd)
+ code = os.system(cmd) # ty:ignore[deprecated]
if code != 0 and rememberFailure:
print("\n[repro] Setting last failure code to %d\n" % code)
lastFailureCode = code
return code
+antOptions: str
+branchFromLog: str | None
+revisionFromLog: str | None
+
+
def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:
global revisionFromLog
global branchFromLog
@@ -152,7 +157,7 @@ def fetchAndParseJenkinsLog(url: str, numRetries: int) -> dict[str, str]:
return tests
-def prepareWorkspace(useGit: bool, gitRef: str | None):
+def prepareWorkspace(useGit: bool, gitRef: str | None) -> None:
global gitCheckoutSucceeded
if useGit:
code = run("git fetch")
@@ -178,7 +183,7 @@ def prepareWorkspace(useGit: bool, gitRef: str | None):
raise RuntimeError('ERROR: "ant clean" failed. See above.')
-def groupTestsByModule(tests: dict[str, str]):
+def groupTestsByModule(tests: dict[str, str]) -> dict[str, set[str]]:
modules: dict[str, set[str]] = {}
for dir, _, files in os.walk("."):
for file in files:
@@ -200,7 +205,7 @@ def groupTestsByModule(tests: dict[str, str]):
return modules
-def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]):
+def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]) -> None:
cwd = os.getcwd()
testCmdline = 'ant test-nocompile -Dtests.dups=%d -Dtests.maxfailures=%d -Dtests.class="%s" -Dtests.showOutput=onerror %s %s'
for module in modules:
@@ -218,7 +223,7 @@ def runTests(testIters: int, modules: dict[str, set[str]], tests: dict[str, str]
os.chdir(cwd)
-def printAndMoveReports(testIters: int, newSubDir: str, location: str):
+def printAndMoveReports(testIters: int, newSubDir: str, location: str) -> dict[str, int]:
failures: dict[str, int] = {}
for start in ("lucene/build", "solr/build"):
for dir, _, files in os.walk(start):
@@ -245,7 +250,7 @@ def printAndMoveReports(testIters: int, newSubDir: str, location: str):
return failures
-def getLocalGitBranch():
+def getLocalGitBranch() -> str:
origGitBranch = runOutput("git rev-parse --abbrev-ref HEAD")
if origGitBranch == "HEAD": # In detached HEAD state
origGitBranch = runOutput("git rev-parse HEAD") # Use the SHA when not on a branch
@@ -253,7 +258,7 @@ def getLocalGitBranch():
return origGitBranch
-def main():
+def main() -> None:
config = readConfig()
tests = fetchAndParseJenkinsLog(config.url, numRetries=2)
localGitBranch = None
diff --git a/dev-tools/scripts/requirements.txt b/dev-tools/scripts/requirements.txt
index 1b834d5b70e..44498c4344c 100644
--- a/dev-tools/scripts/requirements.txt
+++ b/dev-tools/scripts/requirements.txt
@@ -2,6 +2,8 @@
Jinja2==3.1.6
# parsing and processing of releaseWizard.yaml
PyYAML==6.0.3
+# types
+types-PyYAML==6.0.12.20260518
# international holidays in releaseWizard
holidays==0.95
# calendar processing in releaseWizard
@@ -13,6 +15,6 @@ PyGithub==2.9.1
# JIRA processing in githubPRs
jira==3.10.5
# type-checking in "make lint"
-basedpyright==1.38.4
+ty==0.0.46
# linting in "make lint"
ruff==0.15.15
diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py
index 7cbabf22abc..9e608b68484 100644
--- a/dev-tools/scripts/scriptutil.py
+++ b/dev-tools/scripts/scriptutil.py
@@ -28,7 +28,7 @@ from typing import Self, override
class Version:
- def __init__(self, major: int, minor: int, bugfix: int, prerelease: int):
+ def __init__(self, major: int, minor: int, bugfix: int, prerelease: int) -> None:
super().__init__()
self.major = major
self.minor = minor
@@ -39,7 +39,7 @@ class Version:
self.constant = "LUCENE_%d_%d_%d" % (self.major, self.minor, self.bugfix)
@classmethod
- def parse(cls, value: str):
+ def parse(cls, value: str) -> "Version":
match = re.search(r"(\d+)\.(\d+).(\d+)(.1|.2)?", value)
if match is None:
raise argparse.ArgumentTypeError("Version argument must be of format x.y.z(.1|.2)?")
@@ -48,10 +48,10 @@ class Version:
return Version(*parts)
@override
- def __str__(self):
+ def __str__(self) -> str:
return self.dot
- def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\."):
+ def make_previous_matcher(self, prefix: str = "", suffix: str = "", sep: str = "\\.") -> Pattern:
if self.is_bugfix_release():
pattern = "%s%s%s%s%d" % (self.major, sep, self.minor, sep, self.bugfix - 1)
elif self.is_minor_release():
@@ -61,30 +61,30 @@ class Version:
return re.compile(prefix + "(" + pattern + ")" + suffix)
- def is_bugfix_release(self):
+ def is_bugfix_release(self) -> bool:
return self.bugfix != 0
- def is_minor_release(self):
+ def is_minor_release(self) -> bool:
return self.bugfix == 0 and self.minor != 0
- def is_major_release(self):
+ def is_major_release(self) -> bool:
return self.bugfix == 0 and self.minor == 0
- def on_or_after(self, other: Self):
+ def on_or_after(self, other: Self) -> bool:
return self.major > other.major or (
self.major == other.major and (self.minor > other.minor or (self.minor == other.minor and (self.bugfix > other.bugfix or (self.bugfix == other.bugfix and self.prerelease >= other.prerelease))))
)
- def gt(self, other: Self):
+ def gt(self, other: Self) -> bool:
return self.major > other.major or (self.major == other.major and self.minor > other.minor) or (self.major == other.major and self.minor == other.minor and self.bugfix > other.bugfix)
- def is_back_compat_with(self, other: Self):
+ def is_back_compat_with(self, other: Self) -> bool:
if not self.on_or_after(other):
raise Exception("Back compat check disallowed for newer version: %s < %s" % (self, other))
return other.major + 1 >= self.major
-def run(cmd: str, cwd: str | None = None):
+def run(cmd: str, cwd: str | None = None) -> str:
try:
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
@@ -93,7 +93,7 @@ def run(cmd: str, cwd: str | None = None):
return output.decode("utf-8")
-def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None):
+def update_file(filename: str, line_re: Pattern[str], edit: Callable[[list[str], Match[str], str], bool | None], append: Callable[[list[str], bool], bool] | None = None) -> bool:
infile = open(filename)
buffer: list[str] = []
@@ -123,7 +123,7 @@ class BranchType(Enum):
release = 3
-def find_branch_type():
+def find_branch_type() -> BranchType:
output = subprocess.check_output("git status", shell=True)
for line in output.split(b"\n"):
if line.startswith(b"On branch "):
@@ -141,7 +141,7 @@ def find_branch_type():
raise Exception("Cannot run %s on feature branch" % sys.argv[0].rsplit("/", 1)[-1])
-def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True):
+def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_clean: bool = True) -> None:
if not quiet:
print("Downloading %s" % urlString)
startTime = time.time()
@@ -164,7 +164,7 @@ def download(name: str, urlString: str, tmpDir: str, quiet: bool = False, force_
print(" %.1f MB in %.2f sec (%.1f MB/sec)" % (sizeMB, t, sizeMB / t))
-def attemptDownload(urlString: str, fileName: str):
+def attemptDownload(urlString: str, fileName: str) -> None:
fIn = urllib.request.urlopen(urlString)
fOut = open(fileName, "wb")
success = False
@@ -184,7 +184,7 @@ def attemptDownload(urlString: str, fileName: str):
os.remove(fileName)
-def find_current_version():
+def find_current_version() -> str:
version_prop_re = re.compile(r"version\.base=(.*)")
script_path = os.path.dirname(os.path.realpath(__file__))
top_level_dir = os.path.join(Path("%s/" % script_path).resolve(), os.path.pardir, os.path.pardir)
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index 4b1e8799db7..25baa7be113 100755
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -47,10 +47,10 @@ BASE_JAVA_VERSION = "25"
# tested on Linux and on Cygwin under Windows 7.
cygwin = platform.system().lower().startswith("cygwin")
-cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else ""
+cygwinWindowsRoot = os.popen("cygpath -w /").read().strip().replace("\\", "/") if cygwin else "" # ty:ignore[deprecated]
-def unshortenURL(url: str):
+def unshortenURL(url: str) -> str:
parsed = urllib.parse.urlparse(url)
if parsed[0] in ("http", "https"):
h = http.client.HTTPConnection(parsed.netloc)
@@ -71,7 +71,7 @@ reHREF = re.compile('<a href="(.*?)">(.*?)</a>')
FORCE_CLEAN = True
-def getHREFs(urlString: str):
+def getHREFs(urlString: str) -> list[tuple[str, str]]:
# Deref any redirects
while True:
url = urllib.parse.urlparse(urlString)
@@ -103,7 +103,7 @@ def getHREFs(urlString: str):
return links
-def load(urlString: str):
+def load(urlString: str) -> str:
try:
content = urllib.request.urlopen(urlString).read().decode("utf-8")
except Exception as e:
@@ -112,14 +112,14 @@ def load(urlString: str):
return content
-def noJavaPackageClasses(desc: str, file: str):
+def noJavaPackageClasses(desc: str, file: str) -> None:
with zipfile.ZipFile(file) as z2:
for name2 in z2.namelist():
if name2.endswith(".class") and (name2.startswith("java/") or name2.startswith("javax/")):
raise RuntimeError('%s contains sheisty class "%s"' % (desc, name2))
-def decodeUTF8(bytes: bytes):
+def decodeUTF8(bytes: bytes) -> str:
return codecs.getdecoder("UTF-8")(bytes)[0]
@@ -128,7 +128,7 @@ NOTICE_FILE_NAME = "META-INF/NOTICE.txt"
LICENSE_FILE_NAME = "META-INF/LICENSE.txt"
-def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):
+def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str) -> None:
with zipfile.ZipFile(jarFile, "r") as z:
for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME):
try:
@@ -186,11 +186,11 @@ def checkJARMetaData(desc: str, jarFile: str, gitRevision: str, version: str):
raise RuntimeError("%s: %s contents doesn't match main LICENSE.txt" % (desc, LICENSE_FILE_NAME))
-def normSlashes(path: str):
+def normSlashes(path: str) -> str:
return path.replace(os.sep, "/")
-def checkAllJARs(topDir: str, gitRevision: str, version: str):
+def checkAllJARs(topDir: str, gitRevision: str, version: str) -> None:
print(" verify JAR metadata/identity/no javax.* or java.* classes...")
for root, _, files in os.walk(topDir):
normRoot = normSlashes(root)
@@ -205,7 +205,7 @@ def checkAllJARs(topDir: str, gitRevision: str, version: str):
checkJARMetaData('JAR file "%s"' % fullPath, fullPath, gitRevision, version)
-def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str):
+def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFile: str) -> None:
print(" test basics...")
ents = getDirEntries(urlString)
artifact = None
@@ -305,7 +305,7 @@ def checkSigs(urlString: str, version: str, tmpDir: str, isSigned: bool, keysFil
print(" GPG: %s" % line.strip())
-def testChanges(version: str, changesURLString: str):
+def testChanges(version: str, changesURLString: str) -> None:
print(" check changes HTML...")
changesURL = None
for text, subURL in getDirEntries(changesURLString):
@@ -319,7 +319,7 @@ def testChanges(version: str, changesURLString: str):
checkChangesContent(s, version, changesURL, True)
-def testChangesText(dir: str, version: str):
+def testChangesText(dir: str, version: str) -> None:
"Checks all CHANGES.txt under this dir."
for root, _, files in os.walk(dir):
# NOTE: O(N) but N should be smallish:
@@ -334,7 +334,7 @@ reUnderbarNotDashHTML = re.compile(r"<li>(\s*(LUCENE)_\d\d\d\d+)")
reUnderbarNotDashTXT = re.compile(r"\s+((LUCENE)_\d\d\d\d+)", re.MULTILINE)
-def checkChangesContent(s: str, version: str, name: str, isHTML: bool):
+def checkChangesContent(s: str, version: str, name: str, isHTML: bool) -> None:
currentVersionTuple = versionToTuple(version, name)
if isHTML and s.find("Release %s" % version) == -1:
@@ -371,7 +371,7 @@ def checkChangesContent(s: str, version: str, name: str, isHTML: bool):
release = text[8:].strip()
seenText.clear()
releaseTuple = versionToTuple(release, name)
- if releaseTuple > currentVersionTuple:
+ if releaseTuple > currentVersionTuple: # ty:ignore[unsupported-operator]
raise RuntimeError("Future release %s is greater than %s in %s" % (release, version, name))
if id in seenIDs:
raise RuntimeError('%s has duplicate section "%s" under release "%s"' % (name, text, release))
@@ -384,7 +384,7 @@ def checkChangesContent(s: str, version: str, name: str, isHTML: bool):
reVersion = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?\s*(-alpha|-beta|final|RC\d+)?\s*(?:\[.*\])?", re.IGNORECASE)
-def versionToTuple(version: str, name: str):
+def versionToTuple(version: str, name: str) -> tuple[int | str | Any, ...]:
versionMatch = reVersion.match(version)
if versionMatch is None:
raise RuntimeError("Version %s in %s cannot be parsed" % (version, name))
@@ -405,7 +405,7 @@ def versionToTuple(version: str, name: str):
reUnixPath = re.compile(r'\b[a-zA-Z_]+=(?:"(?:\\"|[^"])*"' + "|(?:\\\\.|[^\"'\\s])*" + r"|'(?:\\'|[^'])*')" + r'|(/(?:\\.|[^"\'\s])*)' + r'|("/(?:\\.|[^"])*")' + r"|('/(?:\\.|[^'])*')")
-def unix2win(matchobj: re.Match[str]):
+def unix2win(matchobj: re.Match[str]) -> str:
if matchobj.group(1) is not None:
return cygwinWindowsRoot + matchobj.group()
if matchobj.group(2) is not None:
@@ -415,7 +415,7 @@ def unix2win(matchobj: re.Match[str]):
return matchobj.group()
-def cygwinifyPaths(command: str):
+def cygwinifyPaths(command: str) -> str:
# The problem: Native Windows applications running under Cygwin can't
# handle Cygwin's Unix-style paths. However, environment variable
# values are automatically converted, so only paths outside of
@@ -426,7 +426,7 @@ def cygwinifyPaths(command: str):
return command
-def printFileContents(fileName: str):
+def printFileContents(fileName: str) -> None:
# Assume log file was written in system's default encoding, but
# even if we are wrong, we replace errors ... the ASCII chars
# (which is what we mostly care about eg for the test seed) should
@@ -443,17 +443,17 @@ def printFileContents(fileName: str):
print()
-def run(command: str, logFile: str):
+def run(command: str, logFile: str) -> None:
if cygwin:
command = cygwinifyPaths(command)
- if os.system("%s > %s 2>&1" % (command, logFile)):
+ if os.system("%s > %s 2>&1" % (command, logFile)): # ty:ignore[deprecated]
logPath: str = str(Path(logFile).resolve())
print('\ncommand "%s" failed:' % command)
printFileContents(logFile)
raise RuntimeError('command "%s" failed; see log file %s' % (command, logPath))
-def verifyDigests(artifact: str, urlString: str, tmpDir: str):
+def verifyDigests(artifact: str, urlString: str, tmpDir: str) -> None:
print(" verify sha512 digest")
sha512Expected, t = load(urlString + ".sha512").strip().split()
if t != "*" + artifact:
@@ -472,7 +472,7 @@ def verifyDigests(artifact: str, urlString: str, tmpDir: str):
raise RuntimeError("SHA512 digest mismatch for %s: expected %s but got %s" % (artifact, sha512Expected, sha512Actual))
-def getDirEntries(urlString: str):
+def getDirEntries(urlString: str) -> list[tuple[str, str]]:
if urlString.startswith("file:/") and not urlString.startswith("file://"):
# stupid bogus ant URI
urlString = "file:///" + urlString[6:]
@@ -498,7 +498,7 @@ def getDirEntries(urlString: str):
raise RuntimeError("could not enumerate %s" % (urlString))
-def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str):
+def unpackAndVerify(java: Any, tmpDir: str, artifact: str, gitRevision: str, version: str, testArgs: str) -> str:
destDir = "%s/unpack" % tmpDir
if os.path.exists(destDir):
shutil.rmtree(destDir)
@@ -526,7 +526,7 @@ LUCENE_NOTICE = None
LUCENE_LICENSE = None
-def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):
+def is_in_list(in_folder: list[str], files: list[str], indent: int = 4) -> None:
for fileName in files:
print("%sChecking %s" % (" " * indent, fileName))
found = False
@@ -538,7 +538,7 @@ def is_in_list(in_folder: list[str], files: list[str], indent: int = 4):
raise RuntimeError('file "%s" is missing' % fileName)
-def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str):
+def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str, version: str, testArgs: str) -> None:
global LUCENE_NOTICE
global LUCENE_LICENSE
@@ -633,13 +633,13 @@ def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str,
if isSrc:
print(" make sure no JARs/WARs in src dist...")
- lines = os.popen("find . -name \\*.jar -not -name \\*-api.jar").readlines()
+ lines = os.popen("find . -name \\*.jar -not -name \\*-api.jar").readlines() # ty:ignore[deprecated]
if len(lines) != 0:
print(" FAILED:")
for line in lines:
print(" %s" % line.strip())
raise RuntimeError("source release has JARs...")
- lines = os.popen("find . -name \\*.war").readlines()
+ lines = os.popen("find . -name \\*.war").readlines() # ty:ignore[deprecated]
if len(lines) != 0:
print(" FAILED:")
for line in lines:
@@ -678,7 +678,7 @@ def verifyUnpacked(java: Any, artifact: str, unpackPath: str, gitRevision: str,
testChangesText(".", version)
-def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str):
+def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jdk: str) -> None:
if os.path.exists("index"):
shutil.rmtree("index") # nuke any index from any previous iteration
@@ -727,11 +727,11 @@ def testDemo(run_java: Callable[[str, str], None], isSrc: bool, version: str, jd
raise RuntimeError('wrong version from CheckIndex: got "%s" but expected "%s"' % (actualVersion, version))
-def removeTrailingZeros(version: str):
+def removeTrailingZeros(version: str) -> str:
return re.sub(r"(\.0)*$", "", version)
-def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str):
+def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSigned: bool, keysFile: str) -> None:
print(" download artifacts")
artifacts: list[str] = []
artifactsURL = "%s/lucene/maven/org/apache/lucene/" % baseURL
@@ -753,7 +753,7 @@ def checkMaven(baseURL: str, tmpDir: str, gitRevision: str, version: str, isSign
checkAllJARs("%s/maven/org/apache/lucene" % tmpDir, gitRevision, version)
-def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):
+def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str) -> list[str]:
distribution = "lucene-%s.tgz" % version
if not os.path.exists("%s/%s" % (tmpDir, distribution)):
distURL = "%s/lucene/%s" % (baseURL, distribution)
@@ -767,13 +767,13 @@ def getBinaryDistFiles(tmpDir: str, version: str, baseURL: str):
print(" unpack %s..." % distribution)
unpackLogFile = "%s/unpack-%s-getBinaryDistFiles.log" % (tmpDir, distribution)
run("tar xzf %s/%s" % (tmpDir, distribution), unpackLogFile)
- distributionFiles: list[str] = []
+ distributionFiles = []
for root, _, files in os.walk(destDir):
distributionFiles.extend([os.path.join(root, file) for file in files])
return distributionFiles
-def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):
+def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str) -> None:
print(" check for javadoc and sources artifacts...")
for artifact in artifacts:
if artifact.endswith(version + ".jar"):
@@ -785,8 +785,8 @@ def checkJavadocAndSourceArtifacts(artifacts: list[str], version: str):
raise RuntimeError("missing: %s" % sourcesJar)
-def getZipFileEntries(fileName: str):
- entries: list[str] = []
+def getZipFileEntries(fileName: str) -> list[str]:
+ entries = []
with zipfile.ZipFile(fileName) as zf:
for zi in zf.infolist():
entries.append(zi.filename)
@@ -795,7 +795,7 @@ def getZipFileEntries(fileName: str):
return entries
-def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str):
+def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], version: str) -> None:
print(" verify that Maven artifacts are same as in the binary distribution...")
reJarWar = re.compile(r"%s\.[wj]ar$" % version) # exclude *-javadoc.jar and *-sources.jar
distFilenames: dict[str, str] = dict()
@@ -812,7 +812,7 @@ def checkIdenticalMavenArtifacts(distFiles: list[str], artifacts: list[str], ver
raise RuntimeError("Maven artifact %s is not identical to %s in lucene binary distribution" % (artifact, distFilenames[artifactFilename]))
-def verifyMavenDigests(artifacts: list[str]):
+def verifyMavenDigests(artifacts: list[str]) -> None:
print(" verify Maven artifacts' md5/sha1 digests...")
reJarWarPom = re.compile(r"\.(?:[wj]ar|pom)$")
for artifactFile in [a for a in artifacts if reJarWarPom.search(a)]:
@@ -842,7 +842,7 @@ def verifyMavenDigests(artifacts: list[str]):
raise RuntimeError("SHA1 digest mismatch for %s: expected %s but got %s" % (artifactFile, sha1Expected, sha1Actual))
-def getPOMcoordinate(treeRoot: ET.Element):
+def getPOMcoordinate(treeRoot: ET.Element) -> tuple[str, str, str, str]:
namespace = "{http://maven.apache.org/POM/4.0.0}"
groupId = treeRoot.find("%sgroupId" % namespace)
if groupId is None:
@@ -872,7 +872,7 @@ def getPOMcoordinate(treeRoot: ET.Element):
return groupId, artifactId, packaging, version
-def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):
+def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str) -> None:
print(" verify maven artifact sigs", end=" ")
# Set up clean gpg world; import keys file:
@@ -904,14 +904,14 @@ def verifyMavenSigs(tmpDir: str, artifacts: list[str], keysFile: str):
print()
-def print_warnings_in_file(file: str):
+def print_warnings_in_file(file: str) -> None:
with open(file) as f:
for line in f.readlines():
if line.lower().find("warning") != -1 and line.find("WARNING: This key is not certified with a trusted signature") == -1 and line.find("WARNING: using insecure memory") == -1:
print(" GPG: %s" % line.strip())
-def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):
+def verifyPOMperBinaryArtifact(artifacts: list[str], version: str) -> None:
print(" verify that each binary artifact has a deployed POM...")
reBinaryJarWar = re.compile(r"%s\.[jw]ar$" % re.escape(version))
for artifact in [a for a in artifacts if reBinaryJarWar.search(a)]:
@@ -920,7 +920,7 @@ def verifyPOMperBinaryArtifact(artifacts: list[str], version: str):
raise RuntimeError("missing: POM for %s" % artifact)
-def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):
+def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str) -> None:
"""Verify that each POM's coordinate (drawn from its content) matches
its filepath, and verify that the corresponding artifact exists.
"""
@@ -937,7 +937,7 @@ def verifyDeployedPOMsCoordinates(artifacts: list[str], version: str):
raise RuntimeError("Missing corresponding .%s artifact for POM %s" % (packaging, POM))
-def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None):
+def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions: set[str] | None = None) -> None:
exclude: set[str] = exclusions or set()
for text, subURL in getDirEntries(urlString):
if text not in exclude:
@@ -953,8 +953,11 @@ def crawl(downloadedFiles: list[str], urlString: str, targetDir: str, exclusions
sys.stdout.write(".")
-def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]):
- def _make_runner(java_home: str, is_base_version: bool = False):
+JavaConfig = namedtuple("JavaConfig", "run_java java_home run_alt_javas alt_java_homes alt_java_versions")
+
+
+def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str]) -> JavaConfig:
+ def _make_runner(java_home: str, is_base_version: bool = False) -> tuple[Callable[[str, str], None], str]:
if cygwin:
java_home = subprocess.check_output('cygpath -u "%s"' % java_home, shell=True).decode("utf-8").strip()
cmd_prefix = 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (java_home, java_home, java_home)
@@ -972,7 +975,7 @@ def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str])
elif int(actual_version) < int(BASE_JAVA_VERSION):
parser.error("got wrong version for java %s, less than base version %s:\n%s" % (actual_version, BASE_JAVA_VERSION, s))
- def run_java(cmd: str, logfile: str):
+ def run_java(cmd: str, logfile: str) -> None:
run("%s; %s" % (cmd_prefix, cmd), logfile)
return run_java, actual_version
@@ -989,15 +992,14 @@ def make_java_config(parser: argparse.ArgumentParser, alt_java_homes: list[str])
run_alt_javas.append(run_alt_java)
alt_java_versions.append(version)
- jc = namedtuple("JavaConfig", "run_java java_home run_alt_javas alt_java_homes alt_java_versions")
- return jc(run_java, java_home, run_alt_javas, alt_java_homes, alt_java_versions)
+ return JavaConfig(run_java, java_home, run_alt_javas, alt_java_homes, alt_java_versions)
version_re = re.compile(r"(\d+\.\d+\.\d+(-ALPHA|-BETA)?)")
revision_re = re.compile(r"rev-([a-f\d]+)")
-def parse_config():
+def parse_config() -> argparse.Namespace:
epilogue = textwrap.dedent("""
Example usage:
python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/lucene/lucene-9.0.0-RC1-rev-c7510a0...
@@ -1053,7 +1055,7 @@ reVersion1 = re.compile(r"\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<", re.IGNORECAS
reVersion2 = re.compile(r"-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.", re.IGNORECASE)
-def getAllLuceneReleases():
+def getAllLuceneReleases() -> list[tuple[int, ...]]:
s = load("https://archive.apache.org/dist/lucene/java")
releases: set[tuple[int, ...]] = set()
@@ -1074,7 +1076,7 @@ def getAllLuceneReleases():
return releaseList
-def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str):
+def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str) -> None:
print(" find all past Lucene releases...")
allReleases = getAllLuceneReleases()
# for tup in allReleases:
@@ -1146,7 +1148,7 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion: str, unpackPath: str)
print(" success!")
-def main():
+def main() -> None:
c = parse_config()
# Pick <major>.<minor> part of version and require script to be from same branch
@@ -1160,7 +1162,7 @@ def main():
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, c.local_keys, " ".join(c.test_args), downloadOnly=c.download_only)
-def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False):
+def smokeTest(java: Any, baseURL: str, gitRevision: str, version: str, tmpDir: str, isSigned: bool, local_keys: str | None, testArgs: str, downloadOnly: bool = False) -> None:
startTime = datetime.datetime.now()
# Tests annotated @Nightly are more resource-intensive but often cover