OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
ci: propose upstream service upgrade pull requests (#2371)
Edwin Jose27dd7ffac377
diff --git a/.github/workflows/lint-backend.yml b/.github/workflows/lint-backend.ymlindex 3adc3916f..dbaa3a064 100644--- a/.github/workflows/lint-backend.yml+++ b/.github/workflows/lint-backend.yml@@ -5,6 +5,7 @@ on:paths:- 'src/**/*.py'- 'tests/**/*.py'+ - 'scripts/ci/**/*.py'- 'pyproject.toml'- 'uv.lock'- '.github/workflows/lint-backend.yml'@@ -87,3 +88,18 @@ jobs:if [ "${#test_files[@]}" -gt 0 ]; thenuv run mypy "${test_files[@]}"fi+ scripts_files=()+ for f in ${{ steps.changed.outputs.files }}; do+ case "$f" in+ scripts/*) scripts_files+=("$f") ;;+ esac+ done+ if [ "${#scripts_files[@]}" -gt 0 ]; then+ uv run mypy "${scripts_files[@]}"+ fi++ - name: Run scripts/ci tests+ if: |+ contains(steps.changed.outputs.files, 'scripts/ci/')+ run: |+ uv run python3 scripts/ci/test_update_upstream_version.py -vdiff --git a/.github/workflows/upstream-version-prs.yml b/.github/workflows/upstream-version-prs.ymlnew file mode 100644index 000000000..c24a0cde7--- /dev/null+++ b/.github/workflows/upstream-version-prs.yml@@ -0,0 +1,61 @@+name: Propose upstream service upgrades++on:+ schedule:+ - cron: '17 5 * * *'+ workflow_dispatch:++permissions:+ contents: read++jobs:+ propose:+ if: github.repository == 'langflow-ai/openrag'+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ component: [opensearch, docling, langflow]+ permissions:+ contents: write+ pull-requests: write+ steps:+ - uses: actions/checkout@v7+ with:+ ref: main+ token: ${{ secrets.UPSTREAM_UPDATE_TOKEN || github.token }}++ - uses: actions/setup-python@v7+ with:+ python-version: '3.13'++ - name: Check upstream and update pins+ id: update+ env:+ GH_TOKEN: ${{ secrets.UPSTREAM_UPDATE_TOKEN || github.token }}+ run: python3 scripts/ci/update_upstream_version.py '${{ matrix.component }}'++ - name: Create or update upgrade PR+ if: steps.update.outputs.changed == 'true'+ uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8+ with:+ token: ${{ secrets.UPSTREAM_UPDATE_TOKEN || github.token }}+ branch: automation/update-${{ matrix.component }}+ delete-branch: true+ commit-message: 'chore: update ${{ matrix.component }} to ${{ steps.update.outputs.latest }}'+ title: 'chore: update ${{ matrix.component }} to ${{ steps.update.outputs.latest }}'+ body: |+ Automated upstream version check: `${{ steps.update.outputs.current }}` → `${{ steps.update.outputs.latest }}`.++ Please review compatibility and run the relevant build/integration tests before merging.+ For OpenSearch, this includes the bundled plugins and Dashboards. For Docling Serve,+ review the separately pinned `docling` and `docling-core` versions in the launcher.+ add-paths: |+ Dockerfile+ Dockerfile.langflow+ docker-compose.yml+ kubernetes/helm/openrag/values.yaml+ src/tui/managers/docling_manager.py+ docs/docs/get-started/docker.mdx+ docs/docs/reference/configuration.mdx+ docs/docs/support/troubleshoot.mdxdiff --git a/scripts/ci/test_update_upstream_version.py b/scripts/ci/test_update_upstream_version.pynew file mode 100644index 000000000..49a67b00c--- /dev/null+++ b/scripts/ci/test_update_upstream_version.py@@ -0,0 +1,124 @@+"""Regression tests for the version proposal inputs and guarded edits."""++import tempfile+import unittest+from pathlib import Path+from shutil import copyfile+from unittest.mock import patch++import update_upstream_version as updater+++class UpstreamVersionTests(unittest.TestCase):+ def test_langflow_ignores_prereleases_and_uses_semantic_order(self):+ with tempfile.TemporaryDirectory() as directory:+ dockerfile = Path(directory) / "Dockerfile.langflow"+ dockerfile.write_text("FROM langflowai/langflow:base-0.11.6\n")+ next_page = f"{updater.DOCKER_HUB_API}?page_size=100&name=base-&page=2"+ pages = [+ {"results": [{"name": "base-1.9.0"}, {"name": "base-2.0.0.dev1"}], "next": next_page},+ {"results": [{"name": "base-1.10.0"}, {"name": "latest"}], "next": None},+ ]+ with patch.object(updater, "ROOT", Path(directory)), patch.object(+ updater, "get_json", side_effect=pages+ ):+ self.assertEqual(updater.update_langflow(), ("0.11.6", "1.10.0"))+ self.assertEqual(dockerfile.read_text(), "FROM langflowai/langflow:base-1.10.0\n")++ def test_docling_changes_only_serve_pin(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "src/tui/managers/docling_manager.py"+ path.parent.mkdir(parents=True)+ path.write_text('"docling-serve[ui]==1.26.0"\n"docling-core==2.85.0"\n')+ with patch.object(updater, "ROOT", Path(directory)), patch.object(+ updater, "get_json", return_value={"info": {"version": "1.32.0"}}+ ):+ self.assertEqual(updater.update_docling(), ("1.26.0", "1.32.0"))+ self.assertEqual(path.read_text(), '"docling-serve[ui]==1.32.0"\n"docling-core==2.85.0"\n')++ def test_docling_skips_prerelease(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "src/tui/managers/docling_manager.py"+ path.parent.mkdir(parents=True)+ path.write_text('"docling-serve[ui]==1.26.0"\n')+ with patch.object(updater, "ROOT", Path(directory)), patch.object(+ updater, "get_json", return_value={"info": {"version": "1.33.0rc1"}}+ ):+ self.assertIsNone(updater.update_docling())+ self.assertIn("1.26.0", path.read_text())++ def test_opensearch_skips_release_without_matching_plugin_assets(self):+ with tempfile.TemporaryDirectory() as directory:+ dockerfile = Path(directory) / "Dockerfile"+ dockerfile.write_text("FROM opensearchproject/opensearch:3.6.0 AS upstream_opensearch\n")+ releases = [{"tag_name": "3.7.0", "draft": False, "prerelease": False}]+ with patch.object(updater, "ROOT", Path(directory)), patch.object(+ updater, "get_json", return_value=releases+ ), patch.object(updater, "opensearch_plugin_assets_exist", return_value=False):+ self.assertIsNone(updater.update_opensearch())+ self.assertIn("opensearch:3.6.0", dockerfile.read_text())++ def test_opensearch_updates_image_plugins_dashboards_and_docs(self):+ paths = (+ "Dockerfile",+ "docker-compose.yml",+ "kubernetes/helm/openrag/values.yaml",+ "docs/docs/get-started/docker.mdx",+ "docs/docs/reference/configuration.mdx",+ "docs/docs/support/troubleshoot.mdx",+ )+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ for path in paths:+ target = root / path+ target.parent.mkdir(parents=True, exist_ok=True)+ copyfile(updater.ROOT / path, target)++ # Derive the current pin from the actual Dockerfile so the test+ # stays valid as the repo pin advances past 3.6.0.+ import re as _re+ dockerfile_text = (root / "Dockerfile").read_text()+ current_match = _re.search(+ r"FROM opensearchproject/opensearch:(\d+)\.(\d+)\.(\d+) ",+ dockerfile_text,+ )+ self.assertIsNotNone(current_match, "Could not find OpenSearch pin in Dockerfile")+ major, minor, patch_ = int(current_match.group(1)), int(current_match.group(2)), int(current_match.group(3))+ current = f"{major}.{minor}.{patch_}"+ next_ver = f"{major}.{minor}.{patch_ + 1}"++ releases = [{"tag_name": next_ver, "draft": False, "prerelease": False}]+ with patch.object(updater, "ROOT", root), patch.object(+ updater, "get_json", return_value=releases+ ), patch.object(updater, "opensearch_plugin_assets_exist", return_value=True):+ self.assertEqual(updater.update_opensearch(), (current, next_ver))+ self.assertIn(f"opensearch:{next_ver}", (root / "Dockerfile").read_text())+ self.assertEqual((root / "Dockerfile").read_text().count(f"{next_ver}.0"), 7)+ self.assertIn(f"opensearch-dashboards:{next_ver}", (root / "docker-compose.yml").read_text())+ self.assertIn(f'tag: "{next_ver}"', (root / "kubernetes/helm/openrag/values.yaml").read_text())++ def test_neural_search_asset_name_pre_370(self):+ """Versions before 3.7.0 use the old asset name without -jvector."""+ assets = {"opensearch-neural-search-3.6.0.0.zip", "artifacts.tar.gz", "prometheus-exporter-3.6.0.0.zip"}+ with patch.object(updater, "github_release_assets", return_value=assets):+ self.assertTrue(updater.opensearch_plugin_assets_exist("3.6.0"))++ def test_neural_search_asset_name_370_and_later(self):+ """Versions 3.7.0 and later require the -jvector suffix in the asset name."""+ # Old name should NOT satisfy the check for 3.7.0++ old_assets = {"opensearch-neural-search-3.7.0.0.zip", "artifacts.tar.gz", "prometheus-exporter-3.7.0.0.zip"}+ with patch.object(updater, "github_release_assets", return_value=old_assets):+ self.assertFalse(updater.opensearch_plugin_assets_exist("3.7.0"))++ # New name should satisfy the check for 3.7.0++ new_assets = {"opensearch-neural-search-jvector-3.7.0.0.zip", "artifacts.tar.gz", "prometheus-exporter-3.7.0.0.zip"}+ with patch.object(updater, "github_release_assets", return_value=new_assets):+ self.assertTrue(updater.opensearch_plugin_assets_exist("3.7.0"))++ def test_version_key_rejects_prereleases(self):+ with self.assertRaises(ValueError):+ updater.version_key("1.2.3rc1")+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/ci/update_upstream_version.py b/scripts/ci/update_upstream_version.pynew file mode 100644index 000000000..16c3ac8b5--- /dev/null+++ b/scripts/ci/update_upstream_version.py@@ -0,0 +1,235 @@+"""Propose one compatible upstream service version bump per invocation.++The workflow runs this script separately for each service so a delayed plugin or+an incompatible dependency does not prevent unrelated upgrade PRs.+"""++from __future__ import annotations++import argparse+import json+import os+import re+import sys+from pathlib import Path+from urllib.error import HTTPError, URLError+from urllib.parse import urlsplit+from urllib.request import Request, urlopen++ROOT = Path(__file__).resolve().parents[2]+GITHUB_API = "https://api.github.com"+DOCKER_HUB_API = "https://hub.docker.com/v2/repositories/langflowai/langflow/tags/"+PYPI_API = "https://pypi.org/pypi/docling-serve/json"+VERSION = re.compile(r"^(?:v)?(\d+)\.(\d+)\.(\d+)$")+++def version_key(value: str) -> tuple[int, int, int]:+ match = VERSION.fullmatch(value)+ if not match:+ raise ValueError(f"Not a stable three-part version: {value}")+ major, minor, patch = match.groups()+ return int(major), int(minor), int(patch)+++def get_json(url: str) -> dict[str, object] | list[object]:+ headers = {"Accept": "application/vnd.github+json", "User-Agent": "openrag-version-updater"}+ parsed_url = urlsplit(url)+ if (+ parsed_url.scheme == "https"+ and parsed_url.hostname == "api.github.com"+ and os.environ.get("GH_TOKEN")+ ):+ headers["Authorization"] = f"Bearer {os.environ['GH_TOKEN']}"+ try:+ with urlopen(Request(url, headers=headers), timeout=30) as response:+ return json.load(response)+ except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc:+ raise RuntimeError(f"Could not query {url}: {exc}") from exc+++def get_json_object(url: str) -> dict[str, object]:+ """Like get_json but asserts the response is a JSON object, not an array."""+ result = get_json(url)+ if not isinstance(result, dict):+ raise RuntimeError(f"Expected a JSON object from {url}, got {type(result).__name__}")+ return result+++def get_json_list(url: str) -> list[object]:+ """Like get_json but asserts the response is a JSON array, not an object."""+ result = get_json(url)+ if not isinstance(result, list):+ raise RuntimeError(f"Expected a JSON array from {url}, got {type(result).__name__}")+ return result+++def replace_exact(path: str, old: str, new: str, count: int) -> None:+ file = ROOT / path+ content = file.read_text()+ actual = content.count(old)+ if actual != count:+ raise ValueError(f"Expected {count} occurrences of {old!r} in {path}, found {actual}")+ file.write_text(content.replace(old, new))+++def github_release_assets(repo: str, tag: str) -> set[str]:+ release = get_json_object(f"{GITHUB_API}/repos/{repo}/releases/tags/{tag}")+ assets = release["assets"]+ if not isinstance(assets, list):+ raise RuntimeError(f"Expected 'assets' to be a list in release {tag}")+ return {str(asset["name"]) for asset in assets if isinstance(asset, dict)}+++def opensearch_plugin_assets_exist(version: str) -> bool:+ plugin_version = f"{version}.0"+ # The asset name changed at 3.7.0.0: earlier releases used+ # "opensearch-neural-search-{version}.zip"; from 3.7.0.0 onward it is+ # "opensearch-neural-search-jvector-{version}.zip".+ if version_key(version) >= (3, 7, 0):+ neural_search_asset = f"opensearch-neural-search-jvector-{plugin_version}.zip"+ else:+ neural_search_asset = f"opensearch-neural-search-{plugin_version}.zip"+ expected = (+ ("opensearch-project/opensearch-jvector", "artifacts.tar.gz"),+ ("IBM/neural-search-jvector", neural_search_asset),+ (+ "opensearch-project/opensearch-prometheus-exporter",+ f"prometheus-exporter-{plugin_version}.zip",+ ),+ )+ for repo, asset in expected:+ try:+ if asset not in github_release_assets(repo, plugin_version):+ return False+ except RuntimeError as exc:+ # A missing release is expected while a plugin catches up. Other+ # API failures must fail the job rather than silently hide updates.+ if "HTTP Error 404" in str(exc):+ return False+ raise+ return True+++def update_opensearch() -> tuple[str, str] | None:+ dockerfile = (ROOT / "Dockerfile").read_text()+ match = re.search(r"^FROM opensearchproject/opensearch:(\d+\.\d+\.\d+) ", dockerfile, re.M)+ if not match:+ raise ValueError("Could not find the OpenSearch base image pin")+ current = match.group(1)+ releases = get_json_list(f"{GITHUB_API}/repos/opensearch-project/OpenSearch/releases?per_page=100")+ candidates = sorted(+ (+ str(release["tag_name"]).removeprefix("v")+ for release in releases+ if isinstance(release, dict)+ and not release["draft"]+ and not release["prerelease"]+ and VERSION.fullmatch(str(release["tag_name"]))+ and version_key(str(release["tag_name"])) > version_key(current)+ ),+ key=version_key,+ reverse=True,+ )+ for latest in candidates:+ if not opensearch_plugin_assets_exist(latest):+ print(f"Skipping OpenSearch {latest}: matching plugin assets are unavailable")+ continue+ replace_exact("Dockerfile", f"opensearch:{current}", f"opensearch:{latest}", 1)+ replace_exact("Dockerfile", f"{current}.0", f"{latest}.0", 7)+ replace_exact(+ "docker-compose.yml", f"opensearch-dashboards:{current}", f"opensearch-dashboards:{latest}", 1+ )+ replace_exact("kubernetes/helm/openrag/values.yaml", f'tag: "{current}"', f'tag: "{latest}"', 1)+ for path in (+ "docs/docs/get-started/docker.mdx",+ "docs/docs/reference/configuration.mdx",+ "docs/docs/support/troubleshoot.mdx",+ ):+ replace_exact(+ path,+ f"opensearch-dashboards:{current}",+ f"opensearch-dashboards:{latest}",+ (ROOT / path).read_text().count(f"opensearch-dashboards:{current}"),+ )+ return current, latest+ return None+++def update_docling() -> tuple[str, str] | None:+ path = "src/tui/managers/docling_manager.py"+ content = (ROOT / path).read_text()+ match = re.search(r'"docling-serve\[ui\]==(\d+\.\d+\.\d+)"', content)+ if not match:+ raise ValueError("Could not find the Docling Serve pin")+ current = match.group(1)+ metadata = get_json_object(PYPI_API)+ info = metadata["info"]+ if not isinstance(info, dict):+ raise RuntimeError("Unexpected PyPI response: 'info' is not an object")+ latest = str(info["version"])+ if not VERSION.fullmatch(latest):+ print(f"Skipping Docling Serve: PyPI latest {latest!r} is not a stable release")+ return None+ if version_key(latest) <= version_key(current):+ return None+ replace_exact(path, f'"docling-serve[ui]=={current}"', f'"docling-serve[ui]=={latest}"', 1)+ return current, latest+++def update_langflow() -> tuple[str, str] | None:+ path = "Dockerfile.langflow"+ content = (ROOT / path).read_text()+ match = re.search(r"^FROM langflowai/langflow:base-(\d+\.\d+\.\d+)$", content, re.M)+ if not match:+ raise ValueError("Could not find the Langflow base image pin")+ current = match.group(1)+ # Docker Hub's name filter narrows the listing to base-* tags. Iterate all+ # pages because tags are ordered by update time, not semantic version.+ url = f"{DOCKER_HUB_API}?page_size=100&name=base-"+ candidates: set[str] = set()+ while url:+ data = get_json_object(url)+ results = data["results"]+ if not isinstance(results, list):+ raise RuntimeError("Unexpected Docker Hub response: 'results' is not a list")+ for tag in results:+ if not isinstance(tag, dict):+ continue+ name = str(tag["name"])+ if name.startswith("base-") and VERSION.fullmatch(name[5:]):+ candidates.add(name[5:])+ next_url = data["next"]+ if next_url is not None and not str(next_url).startswith(DOCKER_HUB_API):+ raise RuntimeError(f"Unexpected pagination URL from Docker Hub: {next_url!r}")+ url = str(next_url) if next_url is not None else None+ newer = [version for version in candidates if version_key(version) > version_key(current)]+ if not newer:+ return None+ latest = max(newer, key=version_key)+ replace_exact(path, f"langflow:base-{current}", f"langflow:base-{latest}", 1)+ return current, latest+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("component", choices=("opensearch", "docling", "langflow"))+ args = parser.parse_args()+ result = {+ "opensearch": update_opensearch,+ "docling": update_docling,+ "langflow": update_langflow,+ }[args.component]()+ if result:+ current, latest = result+ print(f"{args.component}: {current} -> {latest}")+ else:+ current = latest = ""+ print(f"{args.component}: no compatible newer version found")+ if output := os.environ.get("GITHUB_OUTPUT"):+ with open(output, "a") as file:+ file.write(f"changed={'true' if result else 'false'}\ncurrent={current}\nlatest={latest}\n")+ return 0+++if __name__ == "__main__":+ sys.exit(main())