Apache Solr open-source search software
releaseWizard: summarize dependency changes from solr/licenses (#4966)
David Smiley28aeacffa57e
diff --git a/.github/scripts/generate-renovate-changelog.py b/.github/scripts/generate-renovate-changelog.pydeleted file mode 100644index ab37fd26288..00000000000--- a/.github/scripts/generate-renovate-changelog.py+++ /dev/null@@ -1,272 +0,0 @@-#!/usr/bin/env python3-"""-Generate changelog YAML entry for Renovate dependency update PRs.--This script parses the PR title to extract dependency information,-then generates a changelog YAML file in changelog/unreleased/ with the proper-naming convention and content structure.--Usage:- python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update org.apache.httpcomponents to v1.2.3"-"""--import argparse-import os-import re-import sys-import yaml-from pathlib import Path-from typing import Optional, Tuple---def sanitize_slug(text: str, max_length: int = 50) -> str:- """- Sanitize text to create a valid filename slug.-- - Convert to lowercase- - Replace dots, colons, slashes with dashes- - Replace other special chars with dashes- - Preserve word boundaries- - Truncate to max_length while preserving word boundaries- """- # Convert to lowercase- text = text.lower()-- # Replace colons, slashes, dots with dashes- text = re.sub(r'[:/.]+', '-', text)-- # Replace other special characters with dashes- text = re.sub(r'[^a-z0-9\s-]', '-', text)-- # Replace spaces with dashes- text = re.sub(r'\s+', '-', text)-- # Replace multiple dashes with single dash- text = re.sub(r'-+', '-', text)-- # Remove leading/trailing dashes- text = text.strip('-')-- # Truncate to max_length at word boundary- if len(text) > max_length:- text = text[:max_length]- # Find last dash and truncate there- last_dash = text.rfind('-')- if last_dash > 0:- text = text[:last_dash]- text = text.rstrip('-')-- return text---def parse_pr_title(title: str) -> Tuple[str, Optional[str]]:- """- Parse Renovate PR title to extract dependency name and version.-- Handles patterns like:- - "Update dependency org.junit.jupiter:junit-jupiter to v6"- - "Update dependency com.jayway.jsonpath:json-path to v2.10.0"- - "Update netty to v4.2.6.Final"- - "Update apache.kafka to v3.9.1"- - "Update actions/checkout action to v5"-- Returns:- Tuple of (title_for_changelog, dependency_slug_for_filename)-- Note: The slug excludes the version number so the filename remains stable- across version updates.- """-- # Pattern 1: "Update dependency {group}:{artifact} to {version}"- match = re.match(r'Update dependency (.+?) to (.+?)(?:\s*$|\s*\(|$)', title)- if match:- dep_name = match.group(1)- version = match.group(2).strip()- changelog_title = f"Update {dep_name} to {version}"- # Slug contains only the dependency name, not the version- slug = sanitize_slug(f"Update {dep_name}")- return changelog_title, slug-- # Pattern 2: "Update {owner}/{action} action to {version}"- match = re.match(r'Update ([a-z0-9-]+/[a-z0-9-]+) action to (.+?)(?:\s*$|\s*\(|$)', title)- if match:- action = match.group(1)- version = match.group(2).strip()- changelog_title = f"Update {action} action to {version}"- # Slug contains only the action name, not the version- slug = sanitize_slug(f"Update {action} action")- return changelog_title, slug-- # Pattern 3: "Update {package} to {version}" (short form)- match = re.match(r'Update ([a-z0-9\-_.]+) to (.+?)(?:\s*$|\s*\(|$)', title)- if match:- package = match.group(1)- version = match.group(2).strip()- changelog_title = f"Update {package} to {version}"- # Slug contains only the package name, not the version- slug = sanitize_slug(f"Update {package}")- return changelog_title, slug-- # Fallback: use title as-is if no pattern matches- return title, sanitize_slug(title)---def generate_changelog_entry(- pr_number: int,- changelog_title: str,- pr_url: str = None-) -> dict:- """Generate the changelog YAML entry dict."""-- if pr_url is None:- pr_url = f"https://github.com/apache/solr/pull/{pr_number}"-- return {- 'title': changelog_title,- 'type': 'dependency_update',- 'authors': [- {'name': 'solrbot'}- ],- 'links': [- {- 'name': f'PR#{pr_number}',- 'url': pr_url- }- ]- }---def find_existing_changelog_file(pr_number: int, changelog_dir: str = 'changelog/unreleased') -> Optional[str]:- """Find existing changelog file for this PR, returns path if exists."""- pattern = f"PR#{pr_number}-*.yml"- path = Path(changelog_dir)-- if not path.exists():- return None-- for file in path.glob(f"PR#{pr_number}-*.yml"):- return str(file)-- return None---def delete_old_changelog_files(pr_number: int, changelog_dir: str = 'changelog/unreleased') -> int:- """- Delete all existing changelog files for this PR number.-- This ensures we don't accumulate orphaned files when the PR title/slug changes.-- Returns:- Number of files deleted- """- pattern = f"PR#{pr_number}-*.yml"- path = Path(changelog_dir)-- if not path.exists():- return 0-- deleted_count = 0- for file in path.glob(pattern):- try:- file.unlink()- print(f"Deleted old changelog file: {file}")- deleted_count += 1- except Exception as e:- print(f"Warning: Could not delete file {file}: {e}", file=sys.stderr)-- if deleted_count > 0:- print(f"Deleted {deleted_count} old changelog file(s) for PR#{pr_number}")-- return deleted_count---def should_update_changelog(existing_file: str, new_title: str) -> bool:- """- Check if we need to update the changelog file.-- Updates if the title has changed (version was bumped).- """- if not existing_file or not Path(existing_file).exists():- return False-- try:- with open(existing_file, 'r') as f:- content = yaml.safe_load(f)-- existing_title = content.get('title', '')- return existing_title != new_title- except Exception as e:- print(f"Warning: Could not read existing file {existing_file}: {e}", file=sys.stderr)- return False---def write_changelog_file(filename: str, entry: dict, changelog_dir: str = 'changelog/unreleased') -> None:- """Write the changelog YAML file."""- path = Path(changelog_dir)- path.mkdir(parents=True, exist_ok=True)-- filepath = path / filename-- # Use YAML dumper that preserves order and formatting- with open(filepath, 'w') as f:- yaml.dump(- entry,- f,- default_flow_style=False,- sort_keys=False,- allow_unicode=True- )-- print(f"Created/updated changelog file: {filepath}")---def main():- parser = argparse.ArgumentParser(- description='Generate changelog entry for Renovate PR',- formatter_class=argparse.RawDescriptionHelpFormatter,- epilog="""-Examples:- python3 generate-renovate-changelog.py --pr-number 1234 --pr-title "Update org.apache.httpcomponents to v1.2.3"- python3 generate-renovate-changelog.py --pr-number 3751 --pr-title "Update dependency com.microsoft.onnxruntime:onnxruntime to v1.23.1"- """- )-- parser.add_argument(- '--pr-number',- type=int,- required=True,- help='GitHub PR number'- )- parser.add_argument(- '--pr-title',- required=True,- help='GitHub PR title (from the Renovate bot)'- )- parser.add_argument(- '--changelog-dir',- default='changelog/unreleased',- help='Directory for changelog files (default: changelog/unreleased)'- )-- args = parser.parse_args()-- # Delete any existing changelog files for this PR to ensure a clean slate- # This prevents orphaned files when the PR title/slug changes- delete_old_changelog_files(args.pr_number, args.changelog_dir)-- # Parse the PR title- changelog_title, slug = parse_pr_title(args.pr_title)-- # Generate filename- filename = f"PR#{args.pr_number}-{slug}.yml"-- # Generate the new entry- entry = generate_changelog_entry(args.pr_number, changelog_title)-- # Write the changelog file- write_changelog_file(filename, entry, args.changelog_dir)- return 0---if __name__ == '__main__':- sys.exit(main())diff --git a/.github/workflows/renovate-changelog-prepare.yml b/.github/workflows/renovate-changelog-prepare.ymldeleted file mode 100644index 5dab69c6867..00000000000--- a/.github/workflows/renovate-changelog-prepare.yml+++ /dev/null@@ -1,123 +0,0 @@-name: Generate Renovate Changelog (Stage 1 - Prepare)--# Stage 1: runs with pull_request (no secrets, no write access).-# Checks out BASE repo code only, runs the trusted Python script,-# and saves the generated changelog file + PR metadata as an artifact-# for Stage 2 to pick up and push to the fork branch.-#-# Generation is skipped (SKIP=true in the artifact metadata) when the PR-# carries the 'no-changelog' label, or when a user other than solrbot has-# manually edited changelog/unreleased/ in the PR — in that case the bot-# must not overwrite the manual changes.-on:- pull_request:- types:- - opened- - synchronize- branches:- - main- - 'branch_*'- paths-ignore:- - '.github/**'--concurrency:- group: renovate-changelog-prepare-${{ github.event.pull_request.number }}- cancel-in-progress: true--permissions:- contents: read- pull-requests: read # Needed to list PR commits for the manual-edit check--jobs:- generate:- # Only run for Renovate bot PRs from the expected fork- if: |- github.event.pull_request.user.login == 'solrbot' &&- github.event.pull_request.head.repo.full_name == 'solrbot/apache-_-solr'- runs-on: ubuntu-latest-- steps:- - name: Checkout BASE repository at base branch (NOT fork code)- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- with:- ref: ${{ github.event.pull_request.base.ref }}- repository: ${{ github.repository }}-- - name: Check whether changelog generation should be skipped- id: skip-check- env:- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}- PR_NUMBER: ${{ github.event.pull_request.number }}- REPO: ${{ github.repository }}- HAS_NO_CHANGELOG_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'no-changelog') }}- run: |- set -euo pipefail- skip=false-- if [ "$HAS_NO_CHANGELOG_LABEL" = "true" ]; then- skip=true- echo "::notice::Skipping changelog generation for PR#${PR_NUMBER}: PR has the 'no-changelog' label"- else- # Skip if any PR commit from a user other than solrbot touched- # changelog/unreleased/ — a human has taken over the changelog- # entry and the bot must not overwrite it. Commits with an- # unresolvable login are treated as manual edits (fail safe).- while read -r sha login; do- if [ "$login" != "solrbot" ]; then- if gh api "repos/${REPO}/commits/${sha}" --jq '.files[].filename' | grep -q '^changelog/unreleased/'; then- skip=true- echo "::notice::Skipping changelog generation for PR#${PR_NUMBER}: changelog/unreleased/ was manually edited by '${login}' in commit ${sha}"- break- fi- fi- done < <(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/commits" \- --jq '.[] | "\(.sha) \(.author.login // "unknown")"')- fi-- echo "skip=${skip}" >> "$GITHUB_OUTPUT"-- - name: Set up Python- if: steps.skip-check.outputs.skip != 'true'- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0- with:- python-version: '3.x'-- - name: Install dependencies- if: steps.skip-check.outputs.skip != 'true'- run: python3 -m pip install --quiet pyyaml-- - name: Generate changelog entry- if: steps.skip-check.outputs.skip != 'true'- env:- PR_NUMBER: ${{ github.event.pull_request.number }}- PR_TITLE: ${{ github.event.pull_request.title }}- run: |- python3 .github/scripts/generate-renovate-changelog.py \- --pr-number "$PR_NUMBER" \- --pr-title "$PR_TITLE"-- - name: Assemble artifact- env:- PR_NUMBER: ${{ github.event.pull_request.number }}- HEAD_REF: ${{ github.event.pull_request.head.ref }}- HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}- SKIP: ${{ steps.skip-check.outputs.skip }}- run: |- set -euo pipefail- mkdir -p artifact- if [ "$SKIP" != "true" ] && [ -d changelog/unreleased ]; then- cp -r changelog/unreleased artifact/changelog-unreleased- fi- # Use printf + env vars to avoid shell injection from PR metadata values- printf 'PR_NUMBER=%s\nHEAD_REF=%s\nHEAD_REPO=%s\nSKIP=%s\n' \- "$PR_NUMBER" "$HEAD_REF" "$HEAD_REPO" "$SKIP" \- > artifact/pr-metadata.env- echo "Artifact contents:"; find artifact/ -type f-- - name: Upload artifact- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1- with:- name: renovate-changelog-artifact- path: artifact/- retention-days: 1- if-no-files-found: errordiff --git a/.github/workflows/renovate-changelog-push.yml b/.github/workflows/renovate-changelog-push.ymldeleted file mode 100644index 0e44c9f434e..00000000000--- a/.github/workflows/renovate-changelog-push.yml+++ /dev/null@@ -1,177 +0,0 @@-name: Generate Renovate Changelog (Stage 2 - Push)--# Stage 2: runs after Stage 1 completes, in the base-repo context with access to-# SOLRBOT_GITHUB_TOKEN. Downloads the pre-generated artifact (no fork code executed here),-# validates metadata, and pushes the changelog file to the fork branch.-#-# If Stage 1 flagged SKIP=true in the artifact metadata (PR has the-# 'no-changelog' label, or a user manually edited changelog/unreleased/),-# this workflow leaves the changelog untouched.-on:- workflow_run:- workflows:- - "Generate Renovate Changelog (Stage 1 - Prepare)"- types:- - completed--# Each Stage 2 run corresponds to a unique Stage 1 run (unique workflow_run.id).-# No cancel-in-progress: Stage 1's concurrency already serializes per-PR.-concurrency:- group: renovate-changelog-push-${{ github.event.workflow_run.id }}--permissions:- actions: read # Required to download artifacts from another workflow run- contents: read # Minimal; actual write access to fork uses SOLRBOT_GITHUB_TOKEN PAT--jobs:- push-changelog:- # Only proceed if Stage 1 succeeded for the expected fork.- # Checking head_repository here avoids a spurious artifact-not-found failure- # when Stage 1 ran but skipped its generate job (e.g. non-solrbot PR).- if: |- github.event.workflow_run.conclusion == 'success' &&- github.event.workflow_run.head_repository.full_name == 'solrbot/apache-_-solr'- runs-on: ubuntu-latest-- steps:- - name: Download artifact from Stage 1- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1- with:- name: renovate-changelog-artifact- run-id: ${{ github.event.workflow_run.id }}- github-token: ${{ secrets.GITHUB_TOKEN }}- path: downloaded-artifact/-- - name: Read and validate PR metadata- id: meta- run: |- set -euo pipefail- META_FILE="downloaded-artifact/pr-metadata.env"- if [ ! -f "$META_FILE" ]; then- echo "::error::Metadata file missing from artifact"; exit 1- fi-- # Parse with grep/cut rather than source to avoid executing file content as shell.- # Use || true so a missing key doesn't abort under set -euo pipefail before the- # explicit emptiness check below can emit a meaningful error message.- PR_NUMBER=$(grep '^PR_NUMBER=' "$META_FILE" | cut -d= -f2- || true)- HEAD_REF=$(grep '^HEAD_REF=' "$META_FILE" | cut -d= -f2- || true)- HEAD_REPO=$(grep '^HEAD_REPO=' "$META_FILE" | cut -d= -f2- || true)- SKIP=$(grep '^SKIP=' "$META_FILE" | cut -d= -f2- || true)-- if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_REF" ] || [ -z "$HEAD_REPO" ]; then- echo "::error::Missing required metadata fields (PR_NUMBER, HEAD_REF, or HEAD_REPO)"; exit 1- fi-- # Security: verify this is the expected fork before using SOLRBOT_GITHUB_TOKEN- if [ "$HEAD_REPO" != "solrbot/apache-_-solr" ]; then- echo "::error::Unexpected HEAD_REPO: '$HEAD_REPO'. Expected 'solrbot/apache-_-solr'. Aborting."; exit 1- fi-- # Validate PR_NUMBER is a plain positive integer (prevents injection)- if ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then- echo "::error::PR_NUMBER is not a valid positive integer: '$PR_NUMBER'"; exit 1- fi-- # Validate HEAD_REF using Git's own branch-name rules. This rejects edge cases- # such as '..', '@{', trailing '.lock', and ':' (dangerous in push refspecs)- # while still accepting valid Renovate branch names like renovate/node@lts.- if ! git check-ref-format --branch "$HEAD_REF" > /dev/null 2>&1; then- echo "::error::HEAD_REF is not a valid Git branch name: '$HEAD_REF'"; exit 1- fi-- # SKIP is optional (absent in artifacts from older Stage 1 runs); default to false- if [ "$SKIP" = "true" ]; then- echo "::notice::Stage 1 flagged skip for PR#${PR_NUMBER} — leaving changelog untouched"- else- SKIP=false- fi-- echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"- echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"- echo "head_repo=$HEAD_REPO" >> "$GITHUB_OUTPUT"- echo "skip=$SKIP" >> "$GITHUB_OUTPUT"- echo "Validated: PR#${PR_NUMBER} on ${HEAD_REPO}@${HEAD_REF} (skip=${SKIP})"-- - name: Clone fork branch- if: steps.meta.outputs.skip != 'true'- env:- SOLRBOT_TOKEN: ${{ secrets.SOLRBOT_GITHUB_TOKEN }}- HEAD_REPO: ${{ steps.meta.outputs.head_repo }}- HEAD_REF: ${{ steps.meta.outputs.head_ref }}- run: |- set -euo pipefail- # Store credentials so the token never appears in the command line or process list- git config --global credential.helper store- printf 'https://x-access-token:%[email protected]\n' "$SOLRBOT_TOKEN" > ~/.git-credentials- chmod 600 ~/.git-credentials-- git clone --depth=1 --branch "$HEAD_REF" \- "https://github.com/${HEAD_REPO}.git" \- fork-checkout-- - name: Apply changelog to fork checkout- if: steps.meta.outputs.skip != 'true'- id: apply- env:- PR_NUMBER: ${{ steps.meta.outputs.pr_number }}- run: |- set -euo pipefail- CHANGELOG_DIR="fork-checkout/changelog/unreleased"- ARTIFACT_DIR="downloaded-artifact/changelog-unreleased"-- if [ ! -d "$CHANGELOG_DIR" ]; then- echo "::error::changelog/unreleased not found in fork checkout"; exit 1- fi-- if [ ! -d "$ARTIFACT_DIR" ]; then- echo "::warning::No changelog-unreleased directory in artifact for PR#${PR_NUMBER}"- echo "has_changes=false" >> "$GITHUB_OUTPUT"- exit 0- fi-- # Remove stale PR#NNN-*.yml files (handles slug changes between synchronize events)- find "$CHANGELOG_DIR" -maxdepth 1 -name "PR#${PR_NUMBER}-*.yml" -delete -print-- # Copy the new PR#NNN-*.yml file(s) from the artifact- COPIED=0- for f in "${ARTIFACT_DIR}/PR#${PR_NUMBER}-"*.yml; do- if [ -f "$f" ]; then- cp -v "$f" "$CHANGELOG_DIR/"- COPIED=$((COPIED + 1))- fi- done-- if [ "$COPIED" -eq 0 ]; then- echo "::warning::No PR#${PR_NUMBER}-*.yml file found in artifact"- echo "has_changes=false" >> "$GITHUB_OUTPUT"- else- echo "Copied $COPIED changelog file(s)"- echo "has_changes=true" >> "$GITHUB_OUTPUT"- fi-- - name: Commit and push to fork branch- if: steps.apply.outputs.has_changes == 'true'- env:- HEAD_REF: ${{ steps.meta.outputs.head_ref }}- PR_NUMBER: ${{ steps.meta.outputs.pr_number }}- run: |- set -euo pipefail- cd fork-checkout-- if [ -z "$(git status --porcelain changelog/unreleased/)" ]; then- echo "No changelog changes (already up to date)"- exit 0- fi-- git config user.name "SolrBot"- git config user.email "[email protected]"-- git add changelog/unreleased/- git commit -m "Add changelog entry for PR#${PR_NUMBER}"-- # Credential store (configured in Clone step) provides authentication- git push origin "HEAD:refs/heads/${HEAD_REF}"-- # Remove credentials from disk now that the push is complete- rm -f ~/.git-credentialsdiff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.ymlindex 66fcc1c0481..af43d05b4d5 100644--- a/.github/workflows/validate-changelog.yml+++ b/.github/workflows/validate-changelog.yml@@ -20,7 +20,7 @@ concurrency:jobs:validate-changelog:name: Check changelog entry- # Skip validation for Renovate PRs (solrbot) - they get changelog entries automatically.+ # Skip validation for Renovate PRs (solrbot); dependency changes are summarized at release time.# On label events, only re-run for the 'no-changelog' label; other label churn# (e.g. from the labeler workflow) leaves the existing check result untouched.if: |diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adocindex 70536cc0f04..ac6f0504ff5 100644--- a/dev-docs/changelog.adoc+++ b/dev-docs/changelog.adoc@@ -292,6 +292,13 @@ Example report output (Json or Markdown):and the final post-vote forward-porting. It is integrated into the ReleaseWizard but can also be run standalone. Run with `--help` for usage.+=== 5.5 Dependency changes summary++`dev-tools/scripts/dependencyChanges.py` summarizes the third-party dependency changes+since the previous release by diffing `solr/licenses/`. By default it prints to stdout;+with `--write`, the Release Wizard uses it to write `dependency_update` changelog entries+(one each for added, upgraded, and removed).+== 6. Further Reading* https://github.com/logchange/logchange[Logchange web page]diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.pydeleted file mode 100755index 3286d03cac4..00000000000--- a/dev-tools/scripts/addDepsToChanges.py+++ /dev/null@@ -1,257 +0,0 @@-#!/usr/bin/env python3-# -*- coding: utf-8 -*--# Licensed to the Apache Software Foundation (ASF) under one or more-# contributor license agreements. See the NOTICE file distributed with-# this work for additional information regarding copyright ownership.-# The ASF licenses this file to You under the Apache License, Version 2.0-# (the "License"); you may not use this file except in compliance with-# the License. You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--"""-Script to create changelog YAML entries for solrbot dependency updates-"""-import os-import sys--sys.path.append(os.path.dirname(__file__))-from scriptutil import *--import argparse-import re-import yaml-from pathlib import Path--line_re = re.compile(r"(.*?) (\(branch_\d+x\) )?\(#(\d+)\)$")---class ChangeEntry:- """- Represents one dependency change entry.- Fields:- - pr_num: string PR number (e.g., '3605')- - message: cleaned commit message text to be shown in CHANGES (noise removed)- - author: Git author (e.g., 'solrbot')- """-- def __init__(self, pr_num: str, message: str, author: str):- self.pr_num = pr_num- self.message = message- self.author = author-- def dep_key(self) -> str:- """- Extract a dependency key from the message after 'Update ' and before ' to', '(' or end.- This is used for de-duplication and sorting. Case-insensitive.- """- m = re.search(r"(?i)update\s+(.+?)(?:\s+to\b|\s*\(|$)", self.message)- if m:- return m.group(1).strip()- return self.message.strip()-- def __str__(self) -> str:- # Keep trailing newline to preserve existing blank-line formatting by update_changes- return f"* PR#{self.pr_num}: {self.message} ({self.author})\n"-- def to_yaml_dict(self) -> dict:- """- Convert to a dictionary suitable for YAML serialization.- Extracts JIRA IDs from the title and adds them to links.- """- # Extract JIRA IDs from the message- title, jira_links = extract_jira_issues_from_title(self.message)-- # Build links: JIRA issues first, then PR- links = jira_links.copy() # Start with JIRA links- links.append({- 'name': f'PR#{self.pr_num}',- 'url': f'https://github.com/apache/solr/pull/{self.pr_num}'- })-- return {- 'title': title,- 'type': 'dependency_update',- 'authors': [- {- 'name': self.author- }- ],- 'links': links- }-- def yaml_filename(self) -> str:- """- Generate a filesystem-safe filename for this entry.- Format: PR#####-slug.yaml- Truncates slug on whitespace boundaries, allowing up to 255 chars total.- """- # Clean message for slug- slug = self.message.lower()- # Replace whitespace with single space- slug = re.sub(r'\s+', ' ', slug)- # Remove non-alphanumeric except dashes- slug = re.sub(r'[^a-z0-9-._ ]', '', slug)-- # Calculate available space for slug- # Format: "PR" + pr_num + "-" + slug + ".yaml"- # Typical PR#1234 = 8 chars + "-" = 9 chars, ".yaml" = 5 chars, total overhead = 14 chars- # Most filesystems limit filenames to 255 chars- max_filename_length = 255- overhead = len(f"PR{self.pr_num}-.yaml")- max_slug_length = max_filename_length - overhead-- # Truncate to max length on word boundaries if necessary- if len(slug) > max_slug_length:- # Find the last space within the limit- truncated = slug[:max_slug_length]- last_dash = truncated.rfind(' ')- if last_dash > max_slug_length // 2: # Keep at least half the available space- slug = truncated[:last_dash]- else:- # If no good word boundary, use hard limit and clean up trailing spaces- slug = truncated.rstrip(' ')- else:- # Remove trailing spaces- slug = slug.rstrip(' ')-- return f"PR{self.pr_num}-{slug}.yaml"---def get_prev_release_tag(ver):- """- Based on a given version, compute the git tag for the "previous" version to calculate changes since.- For a major version, we want all solrbot commits since last release, i.e. X-1.Y.Z- For a minor version, we want all solrbot commits since X.Y-1.0- For a patch version, we want all solrbot commits since X.Y.Z-1- """- releases_arr = run('git tag |grep "releases/solr" | cut -c 15-').strip().split("\n")- releases = list(map(lambda x: Version.parse(x), releases_arr))- if ver.is_major_release():- last = releases.pop()- return "releases/solr/%s" % last.dot- if ver.is_minor_release():- return "releases/solr/%s.%s.0" % (ver.major, ver.minor - 1)- if ver.is_bugfix_release():- return "releases/solr/%s.%s.%s" % (ver.major, ver.minor, ver.bugfix - 1)- return None---def read_config():- parser = argparse.ArgumentParser(description='Adds changelog entries in changelog/ folder')- parser.add_argument('--version', type=Version.parse, help='Solr version to add changes to', required=True)- parser.add_argument('--user', default='solrbot', help='Git user to get changes for. Defaults to solrbot')- newconf = parser.parse_args()- return newconf---def get_gitlog_lines(user: str, prev_tag: str):- """- Run git log and return a list of raw subject lines (%s), newest first.- """- output = run('git log --author=' + user + ' --oneline --no-merges --pretty=format:"%s" ' + prev_tag + '..')- return list(filter(None, output.split("\n")))---def parse_gitlog_lines(lines, author: str):- """- Parse raw git log subject lines into ChangeEntry objects.- - Extract pr_num and message- - Strip '(branch_Nx)' noise- - Remove optional leading 'chore(deps): '- - Normalize 'update dependency X' to 'Update X' (case-insensitive)- """- entries = []- for line in lines:- match = line_re.search(line)- if not match:- print("Skipped un-parsable line: %s" % line)- continue- text = match.group(1)- pr_num = match.group(3)- # Clean message noise- msg = text- msg = re.sub(r"^chore\(deps\):\s*", "", msg, flags=re.IGNORECASE)- # Normalize 'update dependency' to 'Update '- msg = re.sub(r"(?i)^update\s+dependenc(y|ies)\s+", "Update ", msg)- entries.append(ChangeEntry(pr_num=pr_num, message=msg, author=author))- return entries---def write_changelog_yaml(entries):- """- Write each ChangeEntry to a YAML file in changelog/unreleased/- """- changelog_dir = Path('changelog/unreleased')-- # Create directory if it doesn't exist- changelog_dir.mkdir(parents=True, exist_ok=True)-- count = 0- for entry in entries:- filename = changelog_dir / entry.yaml_filename()- yaml_data = entry.to_yaml_dict()-- # Write YAML file with proper formatting- with open(filename, 'w') as f:- yaml.dump(yaml_data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)-- print(f"Created: {filename}")- count += 1-- return count---def dedupe_entries(entries):- """- De-duplicate dependency update entries (newest first input) by dep_key.- Keeps the first occurrence for each dependency key.- """- seen = set()- result = []- for e in entries:- key = e.dep_key().lower()- if key not in seen:- seen.add(key)- result.append(e)- return result---def sort_entries(entries):- """Return a new list sorted alphabetically by dependency key (case-insensitive)."""- return sorted(entries, key=lambda e: e.dep_key().lower())---def main():- if not os.path.exists('CHANGELOG.md'):- sys.exit("Tool must be run from the root of a source checkout.")- newconf = read_config()- prev_tag = get_prev_release_tag(newconf.version)- print("Creating changelog YAML entries for dependency updates since git tag %s" % prev_tag)- try:- gitlog_lines = get_gitlog_lines(newconf.user, prev_tag)- entries = parse_gitlog_lines(gitlog_lines, author=newconf.user)- if entries:- deduped = dedupe_entries(entries)- sorted_entries = sort_entries(deduped)- count = write_changelog_yaml(sorted_entries)- print(f"Successfully created {count} changelog YAML entries")- else:- print("No changes found for version %s" % newconf.version.dot)- print("Done")- except subprocess.CalledProcessError:- print("Error running git log - check your --version")- sys.exit(1)---if __name__ == '__main__':- try:- main()- except KeyboardInterrupt:- print('\nReceived Ctrl-C, exiting early')diff --git a/dev-tools/scripts/dependencyChanges.py b/dev-tools/scripts/dependencyChanges.pynew file mode 100644index 00000000000..c5d71a381f4--- /dev/null+++ b/dev-tools/scripts/dependencyChanges.py@@ -0,0 +1,202 @@+#!/usr/bin/env python3+# -*- coding: utf-8 -*-+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Summarizes changes to the third-party dependencies Solr ships, by diffing the+jar checksum files in solr/licenses/ between two git refs (by default: the+previous release tag and HEAD). Jars that moved between the same two versions+are listed together, since they are typically from the same project.++Prints the summary to stdout, or with --write, writes changelog YAML entries.++This is a stopgap until Solr publishes an SBOM per release, which would be the+better source to diff.+"""++import argparse+import os+import re+import subprocess+import sys+from collections import defaultdict++sys.path.append(os.path.dirname(__file__))+from scriptutil import Version, find_current_version++LICENSES_DIR = 'solr/licenses'+DEFAULT_OUTPUT_DIR = 'changelog/unreleased'++# A version starts at the last '-'-delimited segment that looks like a dotted number.+# e.g. log4j-1.2-api-2.25.3, zstd-jni-1.5.6-10, guava-33.4.8-jre+VERSION_SEG_RE = re.compile(r'^\d+\.\d')+# Classifiers that distinguish separate jars of the same artifact and version+CLASSIFIER_RE = re.compile(r'-(tests?|linux-[\w-]+|osx-[\w-]+|windows-[\w-]+)$')+# Test artifacts sharing a LICENSE file with shipped ones (e.g. lucene-test-framework)+TEST_ARTIFACT_RE = re.compile(r'-(tests?|testing|test-framework)$')+++def git(*args):+ return subprocess.run(['git'] + list(args), capture_output=True, text=True, check=True).stdout+++def parse_jar(jar_name):+ """Returns (artifact, version) for a jar name without the '.jar' extension."""+ classifier = ''+ m = CLASSIFIER_RE.search(jar_name)+ if m:+ classifier = m.group(0)+ jar_name = jar_name[:m.start()]+ segs = jar_name.split('-')+ idx = max((i for i, s in enumerate(segs) if i > 0 and VERSION_SEG_RE.match(s)), default=None)+ if idx is None:+ idx = max((i for i, s in enumerate(segs) if i > 0 and s[:1].isdigit()), default=len(segs))+ return '-'.join(segs[:idx]) + classifier, '-'.join(segs[idx:])+++def read_licenses_dir(ref):+ """Returns (artifact -> version, set of LICENSE file prefixes) at the given ref."""+ jars = {}+ license_prefixes = set()+ for name in git('ls-tree', '--name-only', f'{ref}:{LICENSES_DIR}').split():+ if name.endswith('.jar.sha1'):+ artifact, version = parse_jar(name[:-len('.jar.sha1')])+ jars[artifact] = version+ else:+ m = re.match(r'(.+)-LICENSE-.+\.txt$', name)+ if m:+ license_prefixes.add(m.group(1))+ return jars, license_prefixes+++def has_license(artifact, license_prefixes):+ """Whether some '-'-delimited prefix of the artifact has a LICENSE file (mirrors jar-checks.gradle)."""+ name = CLASSIFIER_RE.sub('', artifact)+ while True:+ if name in license_prefixes:+ return True+ prefix = re.sub(r'-[^-]+$', '', name)+ if prefix == name:+ return False+ name = prefix+++def find_previous_release_tag(current):+ cur = Version.parse(current)+ best = None+ for tag in git('tag', '-l', 'releases/solr/*').split():+ m = re.fullmatch(r'releases/solr/(\d+)\.(\d+)\.(\d+)', tag)+ if not m:+ continue+ v = tuple(int(x) for x in m.groups())+ if v < (cur.major, cur.minor, cur.bugfix) and (best is None or v > best[0]):+ best = (v, tag)+ if best is None:+ sys.exit(f'No release tag found before {current}; pass --from')+ return best[1]+++def describe_artifacts(artifacts):+ """e.g. 'asm, asm-tree', or 'jetty-* (23 jars)' when all share the first name segment."""+ if len(artifacts) > 3:+ first = {a.split('-')[0] for a in artifacts}+ if len(first) == 1:+ return f'{first.pop()}-* ({len(artifacts)} jars)'+ return ', '.join(artifacts)+++CATEGORIES = ('added', 'upgraded', 'removed')+++def compute_changes(from_ref, to_ref):+ """Returns category ('added', 'upgraded', 'removed') -> sorted list of human-readable changes."""+ old_jars, old_license_prefixes = read_licenses_dir(from_ref)+ new_jars, new_license_prefixes = read_licenses_dir(to_ref)++ # Only jars with a LICENSE file count: *.sha1 files also cover jars we don't ship (e.g. test+ # dependencies), whereas LICENSE files are only required for shipped ones (since SOLR-15465; older+ # releases have them for non-shipped jars too, making removals from such a release unreliable).+ def shipped(artifact, license_prefixes):+ return has_license(artifact, license_prefixes) and not TEST_ARTIFACT_RE.search(artifact)++ by_transition = defaultdict(list) # (old_version or None, new_version or None) -> artifacts+ for artifact in old_jars.keys() | new_jars.keys():+ old, new = old_jars.get(artifact), new_jars.get(artifact)+ if old == new:+ continue+ if shipped(artifact, new_license_prefixes if new else old_license_prefixes):+ by_transition[(old, new)].append(artifact)++ changes = {c: [] for c in CATEGORIES}+ for (old, new), artifacts in by_transition.items():+ names = describe_artifacts(sorted(artifacts, key=str.lower))+ if old is None:+ changes['added'].append(f'{names} {new}')+ elif new is None:+ changes['removed'].append(f'{names} {old}')+ else:+ changes['upgraded'].append(f'{names} {old} → {new}')+ return {c: sorted(lines, key=str.lower) for c, lines in changes.items()}+++def to_yaml(title, version):+ def quote(s):+ return "'" + s.replace("'", "''") + "'"+ return (f'# Generated by dev-tools/scripts/dependencyChanges.py\n'+ f'title: {quote(title)}\n'+ f'type: dependency_update\n'+ f'authors:\n'+ f' - name: various contributors\n'+ f'links:\n'+ f' - name: solr/licenses\n'+ f' url: https://github.com/apache/solr/tree/releases/solr/{version}/solr/licenses\n')+++def main():+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument('--from', dest='from_ref',+ help='Git ref to compare from (default: the release tag preceding the current version)')+ parser.add_argument('--to', dest='to_ref', default='HEAD', help='Git ref to compare to (default: HEAD)')+ parser.add_argument('--write', nargs='?', const=DEFAULT_OUTPUT_DIR, metavar='DIR',+ help='Write changelog YAML entries (one per category) instead of printing'+ f' (default dir: {DEFAULT_OUTPUT_DIR})')+ args = parser.parse_args()++ version = find_current_version()+ from_ref = args.from_ref or find_previous_release_tag(version)+ changes = compute_changes(from_ref, args.to_ref)+ since = from_ref.rsplit('/', 1)[-1]++ if not args.write:+ print(f'Dependency changes from {from_ref} to {args.to_ref}:')+ for category in CATEGORIES:+ print(f'{category.capitalize()}: ' + ('; '.join(changes[category]) or '(none)'))+ return++ for i, category in enumerate(CATEGORIES, 1): # numbered so the changelog lists them in this order+ path = os.path.join(args.write, f'dependency-changes-{i}-{category}.yml')+ if changes[category]:+ title = f'Third-party dependencies {category} since {since}: ' + '; '.join(changes[category])+ with open(path, 'w', encoding='utf-8') as f:+ f.write(to_yaml(title, version))+ print(f'Wrote {len(changes[category])} {category} dependency changes to {path}')+ elif os.path.exists(path):+ os.remove(path)+ print(f'Removed {path}; no {category} dependency changes')+++if __name__ == '__main__':+ main()diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yamlindex 832b73d7858..1e13c326851 100644--- a/dev-tools/scripts/releaseWizard.yaml+++ b/dev-tools/scripts/releaseWizard.yaml@@ -675,21 +675,21 @@ groups:id: dependency_updates_changestitle: Add dependency updates to changelogdescription: |- Bulk add all 'solrbot' dependency updates since last release.- NOTE: Work in progress to let each Solrbot PR add its own changes to changelog.- This step will be removed once that is done.- Until then, there may be a mix of PRs with and without changes to changelog.+ Add changelog entries summarizing third-party dependencies added, upgraded, and removed since the+ previous release, derived from the jar checksum files in `solr/licenses/`.+ Re-running it overwrites the entries, e.g. if dependencies changed before a respin.+ Run the script without `--write` to just print the summary.depends: clean_git_checkoutcommands: !Commandsroot_folder: '{{ git_checkout_folder }}'- commands_text: We call out to a helper script that compiles SolrBot changes into `changelog/unreleased`+ commands_text: We call out to a helper script that writes `changelog/unreleased/dependency-changes-*.yml`confirm_each_command: truecommands:- !Commandcmd: git checkout {{ release_branch }}stdout: true- !Command- cmd: python3 -u dev-tools/scripts/addDepsToChanges.py --user solrbot --version {{ release_version }}+ cmd: python3 -u dev-tools/scripts/dependencyChanges.py --writetee: true- !Commandcmd: git add changelog && git commit -m "Add dependency updates to changelog for {{ release_version }}" && git push