Apache DataFusion SQL Query Engine
dev: Allow new release version to be automatically updated (#26007)
Nuno Fariada3cb0a2a973
diff --git a/dev/release/README.md b/dev/release/README.mdindex 477e01a068..71714ae1d9 100644--- a/dev/release/README.md+++ b/dev/release/README.md@@ -90,20 +90,16 @@ git push -u apache branch-55 # push branch to apache remote### 2. Prepare PR to Update the Release Version-Update the DataFusion version in the root `Cargo.toml`, in `Cargo.lock` and in the user-documentation. At the time of this writing the documentation files are+Update the DataFusion version using the following Python script:-- `docs/source/download.md`-- `docs/source/user-guide/configs.md`-- `docs/source/user-guide/crate-configuration.md`-- `docs/source/user-guide/example-usage.md`+```shell+# replace NEW_VERSION before executing (e.g., 55.2.0)+python3 dev/update_datafusion_versions.py NEW_VERSION+```-For example, to update the version from `55.1.0` to `55.2.0`:+This updates the DataFusion version across all files, including documentation. The only extra step required is to update the `Cargo.lock` file, using the following command:```shell-# version and lockfile-sed -i '' 's/"55\.1\.0"/"55.2.0"/g' Cargo.toml-sed -i '' 's/55\.1\.0/55.2.0/' docs/source/download.md docs/source/user-guide/{configs,crate-configuration,example-usage}.mdcargo update --workspace```diff --git a/dev/update_datafusion_versions.py b/dev/update_datafusion_versions.pyindex ab27c70a4c..cc7bf1269d 100755--- a/dev/update_datafusion_versions.py+++ b/dev/update_datafusion_versions.py@@ -13,29 +13,6 @@ import argparsefrom pathlib import Pathimport tomlkit-crates = {- 'datafusion-common': 'datafusion/common/Cargo.toml',- 'datafusion-common-runtime': 'datafusion/common-runtime/Cargo.toml',- 'datafusion': 'datafusion/core/Cargo.toml',- 'datafusion-execution': 'datafusion/execution/Cargo.toml',- 'datafusion-expr': 'datafusion/expr/Cargo.toml',- 'datafusion-ffi': 'datafusion/ffi/Cargo.toml',- 'datafusion-functions': 'datafusion/functions/Cargo.toml',- 'datafusion-functions-aggregate': 'datafusion/functions-aggregate/Cargo.toml',- 'datafusion-functions-nested': 'datafusion/functions-nested/Cargo.toml',- 'datafusion-optimizer': 'datafusion/optimizer/Cargo.toml',- 'datafusion-physical-expr': 'datafusion/physical-expr/Cargo.toml',- 'datafusion-physical-expr-common': 'datafusion/physical-expr-common/Cargo.toml',- 'datafusion-physical-plan': 'datafusion/physical-plan/Cargo.toml',- 'datafusion-proto': 'datafusion/proto/Cargo.toml',- 'datafusion-sql': 'datafusion/sql/Cargo.toml',- 'datafusion-sqllogictest': 'datafusion/sqllogictest/Cargo.toml',- 'datafusion-substrait': 'datafusion/substrait/Cargo.toml',- 'datafusion-wasmtest': 'datafusion/wasmtest/Cargo.toml',- 'datafusion-benchmarks': 'benchmarks/Cargo.toml',- 'datafusion-cli': 'datafusion-cli/Cargo.toml',- 'datafusion-examples': 'datafusion-examples/Cargo.toml',-}def update_workspace_version(new_version: str):cargo_toml = 'Cargo.toml'@@ -51,47 +28,15 @@ def update_workspace_version(new_version: str):doc = tomlkit.parse(data)- for crate in crates.keys():- df_dep = doc.get('workspace').get('dependencies', {}).get(crate)+ for crate, df_dep in doc['workspace']['dependencies'].items():+ if not crate.startswith('datafusion'):+ continue# skip crates that pin datafusion using git hashif df_dep is not None and df_dep.get('version') is not None:print(f'updating {crate} dependency in {cargo_toml}')df_dep['version'] = new_version- with open(cargo_toml, 'w') as f:- f.write(tomlkit.dumps(doc))---def update_datafusion_version(cargo_toml: str, new_version: str):- print(f'updating {cargo_toml}')- with open(cargo_toml) as f:- data = f.read()-- doc = tomlkit.parse(data)- pkg = doc.get('package')- if 'workspace' not in pkg['version']:- pkg['version'] = new_version-- with open(cargo_toml, 'w') as f:- f.write(tomlkit.dumps(doc))--def update_downstream_versions(cargo_toml: str, new_version: str):- with open(cargo_toml) as f:- data = f.read()-- doc = tomlkit.parse(data)-- for crate in crates.keys():- df_dep = doc.get('dependencies', {}).get(crate)- # skip crates that pin datafusion using git hash- if df_dep is not None and df_dep.get('version') is not None:- print(f'updating {crate} dependency in {cargo_toml}')- df_dep['version'] = new_version-- df_dep = doc.get('dev-dependencies', {}).get(crate)- if df_dep is not None and df_dep.get('version') is not None:- print(f'updating {crate} dev-dependency in {cargo_toml}')- df_dep['version'] = new_version+ doc['workspace']['package']['version'] = new_versionwith open(cargo_toml, 'w') as f:f.write(tomlkit.dumps(doc))@@ -104,6 +49,7 @@ def update_docs(path: str, new_version: str):fd.seek(0)content = re.sub(r'datafusion\s*=\s*"(.+?)"', f'datafusion = "{new_version}"', content)content = re.sub(r'datafusion\s*=\s*\{\s*version\s*=\s*"(.+?)"', f'datafusion = {{ version = "{new_version}"', content)+ content = re.sub(r'datafusion version \d+\.\d+\.\d+', f'datafusion version {new_version}', content)fd.truncate()fd.write(content)@@ -138,18 +84,11 @@ def main():print(f'Updating workspace in {repo_root} to {new_version}')update_workspace_version(new_version)- print(f'Updating datafusion crate versions in {repo_root} to {new_version}')- for cargo_toml in crates.values():- update_datafusion_version(cargo_toml, new_version)-- print(f'Updating datafusion dependency versions in {repo_root} to {new_version}')- for cargo_toml in crates.values():- update_downstream_versions(cargo_toml, new_version)-update_docs("README.md", new_version)update_docs("docs/source/download.md", new_version)update_docs("docs/source/user-guide/example-usage.md", new_version)update_docs("docs/source/user-guide/crate-configuration.md", new_version)+ update_docs("docs/source/user-guide/configs.md", new_version)update_ci(new_version)