Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(ci): bound dependency downloads and recover through mirrors

efeire344ffdc1d50d
diff --git a/.github/scripts/ci-gradle.init.gradle b/.github/scripts/ci-gradle.init.gradle
new file mode 100644
index 0000000000..e4c5dd2fbd
--- /dev/null
+++ b/.github/scripts/ci-gradle.init.gradle
@@ -0,0 +1,54 @@
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository
+
+def target = System.getenv('DBX_CI_GRADLE_ROOT')
+def backup = System.getenv('DBX_CI_GRADLE_SOURCE') == 'backup'
+def mirror = 'https://maven-central.storage-download.googleapis.com/maven2/'
+def applies = { File directory -> target != null && directory.canonicalPath == target }
+
+beforeSettings { settings ->
+ if (applies(settings.settingsDir)) {
+ if (System.getenv('DBX_CI_GRADLE_PHASE') == 'execute') {
+ gradle.startParameter.offline = true
+ }
+ if (backup) {
+ settings.pluginManagement.repositories.clear()
+ settings.pluginManagement.repositories.maven { url = uri(mirror) }
+ }
+ }
+}
+
+afterProject { project ->
+ if (applies(project.rootDir) && backup) {
+ project.repositories.withType(MavenArtifactRepository).each { repository ->
+ if (repository.url.toString() in ['https://repo.maven.apache.org/maven2/',
+ 'https://repo.maven.apache.org/maven2']) {
+ if (repository.credentials.username || repository.credentials.password) {
+ throw new GradleException('CI mirror switching does not support authenticated Central repositories')
+ }
+ repository.url = uri(mirror)
+ }
+ }
+ }
+}
+
+projectsEvaluated {
+ if (applies(rootProject.rootDir)) {
+ rootProject.tasks.register('dbxCiAcquireDependencies') {
+ doLast {
+ rootProject.allprojects.each { project ->
+ project.configurations.findAll { configuration ->
+ configuration.canBeResolved &&
+ (System.getenv('DBX_CI_GRADLE_SCOPE') != 'jdbc' ||
+ configuration.name in ['compileClasspath', 'runtimeClasspath', 'annotationProcessor', 'shadow'])
+ }.each { configuration ->
+ def artifacts = configuration.incoming.artifactView {
+ componentFilter { identifier -> identifier instanceof ModuleComponentIdentifier }
+ }.files.files
+ println("Acquired ${project.path}:${configuration.name}: ${artifacts.size()} external artifacts")
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/.github/scripts/ci-gradle.py b/.github/scripts/ci-gradle.py
new file mode 100644
index 0000000000..6c2c5aada7
--- /dev/null
+++ b/.github/scripts/ci-gradle.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Acquire Gradle dependencies with bounded recovery, then execute the original command once offline."""
+
+import argparse
+import os
+from pathlib import Path
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from urllib.parse import urlsplit
+
+ROOT = Path(__file__).resolve().parents[2]
+INIT = Path(__file__).with_name("ci-gradle.init.gradle")
+starting_process = False
+pending_cancel = None
+
+
+class Cancelled(BaseException):
+ def __init__(self, signum):
+ self.signum = signum
+
+
+def cancel(signum, _frame):
+ global pending_cancel
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ if starting_process:
+ pending_cancel = signum
+ else:
+ raise Cancelled(signum)
+
+
+def dispose(process):
+ try:
+ os.killpg(process.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ return process.communicate()[0] or ""
+
+
+def run_process(command, cwd, env, budget, capture=True):
+ global starting_process, pending_cancel
+ process = None
+ timed_out = False
+ try:
+ starting_process = True
+ process = subprocess.Popen(command, cwd=cwd, env=env,
+ stdout=subprocess.PIPE if capture else None,
+ stderr=subprocess.STDOUT if capture else None,
+ text=True, start_new_session=True)
+ starting_process = False
+ if pending_cancel is not None:
+ signum, pending_cancel = pending_cancel, None
+ raise Cancelled(signum)
+ try:
+ output = process.communicate(timeout=budget)[0] or ""
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ output = dispose(process)
+ except BaseException:
+ if process is not None:
+ dispose(process)
+ raise
+ finally:
+ starting_process = False
+ if capture:
+ def redact(match):
+ url = urlsplit(match[0])
+ return f"{url.scheme}://{url.hostname}{url.path}"
+ print(re.sub(r"https?://[^\s'\"<>]+", redact, output), end="", flush=True)
+ return process.returncode, output, timed_out
+
+
+def public_maven_url(value):
+ url = urlsplit(value)
+ return (url.scheme == "https" and not url.username and not url.password and
+ not url.query and not url.fragment and url.port in (None, 443) and
+ ((url.hostname == "repo.maven.apache.org" and url.path.startswith("/maven2/")) or
+ (url.hostname == "plugins.gradle.org" and url.path.startswith("/m2/")) or
+ (url.hostname == "maven-central.storage-download.googleapis.com" and url.path.startswith("/maven2/"))))
+
+
+def acquisition_failure(output, timed_out=False):
+ if re.search(r"certificate|PKIX|SSLHandshake|checksum|signature|verification failed|"
+ r"No space left|Permission denied|Could not compile|Compilation failed|"
+ r"Could not find |Could not resolve.*conflict|401\b|404\b", output, re.I):
+ return None
+ section = output.split("* What went wrong:", 1)[-1].split("* Try:", 1)[0]
+ requests = re.findall(r"Could not (?:GET|HEAD) '([^']+)'", section)
+ if timed_out and "* What went wrong:" not in output and not requests:
+ return "transient"
+ if not requests or not all(public_maven_url(url) for url in requests):
+ return None
+ network = re.compile(r"timed out|Connection (?:reset|refused)|UnknownHostException|"
+ r"Temporary failure|Name or service not known|Remote host terminated|"
+ r"Received status code (?:403|429|500|502|503|504)\b", re.I)
+ context = re.compile(r"^(?:A problem occurred configuring |Could not resolve |"
+ r"Could not get resource |Could not (?:GET|HEAD) '|"
+ r"There (?:is|are) \d+ more failure)")
+ details = re.findall(r"^\s*>\s*(.+)$", section, re.M)
+ if not details or any(not context.search(line) and not network.search(line) for line in details):
+ return None
+ if any(re.search(r"Received status code (?!403\b|429\b|500\b|502\b|503\b|504\b)\d+", line)
+ for line in details):
+ return None
+ if re.search(r"Received status code 403\b", section):
+ return "forbidden"
+ return "transient" if timed_out or network.search(section) else None
+
+
+def wrapper_failure(output, timed_out):
+ if re.search(r"certificate|PKIX|checksum|signature|Permission denied|No space left|"
+ r"HTTP response code: (?:401|403|404)", output, re.I):
+ return False
+ return timed_out or bool(re.search(r"timed out|UnknownHostException|Connection (?:reset|refused)|"
+ r"HTTP response code: (?:429|500|502|503|504)", output, re.I))
+
+
+def acquire_and_run(project, scope, command, runner=run_process, sleep=time.sleep):
+ env = {**os.environ, "LC_ALL": "C", "DBX_CI_GRADLE_ROOT": str(project.resolve()),
+ "DBX_CI_GRADLE_SCOPE": scope, "DBX_CI_GRADLE_SOURCE": "primary",
+ "DBX_CI_GRADLE_PHASE": "acquire"}
+ env["GRADLE_OPTS"] = env.get("GRADLE_OPTS", "") + " -Dorg.gradle.daemon=false -Duser.language=en -Duser.country=US"
+ home = Path(env.get("GRADLE_USER_HOME", str(Path.home() / ".gradle"))).expanduser().resolve()
+ env["GRADLE_USER_HOME"] = str(home)
+ init_directory = home / "init.d"
+ init_directory.mkdir(parents=True, exist_ok=True)
+ wrapper = [str(project / "gradlew"), "--no-daemon", "--console=plain"]
+ init_path = None
+ try:
+ with tempfile.NamedTemporaryFile(prefix="dbx-ci-acquire-", suffix=".gradle", dir=init_directory,
+ delete=False) as temporary:
+ init_path = Path(temporary.name)
+ temporary.write(INIT.read_bytes())
+ for attempt in range(3):
+ print(f"Gradle phase=wrapper attempt={attempt + 1} budget=180s", flush=True)
+ code, output, timed_out = runner([*wrapper, "--version"], project, env, 180)
+ if code == 0 and not timed_out:
+ break
+ if attempt == 2 or not wrapper_failure(output, timed_out):
+ return code or 1
+ sleep(2 ** (attempt + 1))
+ acquisition_count = 0
+ for source in ["primary", "backup"]:
+ env["DBX_CI_GRADLE_SOURCE"] = source
+ attempts = 2 if source == "primary" else 3 - acquisition_count
+ for attempt in range(attempts):
+ acquisition_count += 1
+ print(f"Gradle phase=acquire source={source} attempt={attempt + 1} budget=600s", flush=True)
+ code, output, timed_out = runner([*wrapper, "dbxCiAcquireDependencies"], project, env, 600)
+ if code == 0 and not timed_out:
+ env["DBX_CI_GRADLE_PHASE"] = "execute"
+ print(f"Gradle phase=execute source={source} offline=true attempts=1", flush=True)
+ code, _output, _timed_out = runner(command, project, env, None, capture=False)
+ return code
+ reason = acquisition_failure(output, timed_out)
+ if reason is None:
+ return code or 1
+ if source == "primary" and (reason == "forbidden" or attempt + 1 == attempts):
+ break
+ if attempt + 1 == attempts:
+ return code or 1
+ sleep(2)
+ if source == "primary":
+ print("Gradle switching once to Google-hosted Maven Central mirror", flush=True)
+ return 1
+ finally:
+ if init_path is not None:
+ init_path.unlink()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("project", choices=["agents", "jdbc"])
+ parser.add_argument("command", nargs=argparse.REMAINDER)
+ args = parser.parse_args()
+ command = args.command[1:] if args.command[:1] == ["--"] else args.command
+ if not command:
+ parser.error("an original command is required after --")
+ project = ROOT / ("agents" if args.project == "agents" else "plugins/jdbc")
+ signal.signal(signal.SIGINT, cancel)
+ signal.signal(signal.SIGTERM, cancel)
+ try:
+ return acquire_and_run(project, args.project, command)
+ except Cancelled as error:
+ print("Gradle cancelled; no acquisition retry", file=sys.stderr)
+ return 128 + error.signum
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/scripts/ci-gradle.test.py b/.github/scripts/ci-gradle.test.py
new file mode 100644
index 0000000000..dedc853842
--- /dev/null
+++ b/.github/scripts/ci-gradle.test.py
@@ -0,0 +1,222 @@
+import importlib.util
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+from unittest.mock import patch
+
+SPEC = importlib.util.spec_from_file_location("ci_gradle", Path(__file__).with_name("ci-gradle.py"))
+gradle = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(gradle)
+OK = (0, "", False)
+
+
+def failure(status=403, url="https://repo.maven.apache.org/maven2/example/a.pom"):
+ return (1, f"""FAILURE: Build failed with an exception.
+* What went wrong:
+A problem occurred configuring root project 'fixture'.
+> Could not resolve all artifacts for configuration 'classpath'.
+ > Could not resolve example:a:1.
+ > Could not get resource '{url}'.
+ > Could not GET '{url}'.
+ > Received status code {status} from server: Forbidden
+* Try:
+> Run with --stacktrace option to get the stack trace.
+""", False)
+
+
+class AcquisitionTests(unittest.TestCase):
+ def setUp(self):
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.root = Path(self.directory.name)
+ self.project = self.root / "project"
+ self.project.mkdir()
+ self.home = self.root / "gradle-home"
+ self.calls = []
+ self.sleeps = []
+
+ def execute(self, responses):
+ responses = iter(responses)
+
+ def run(command, cwd, env, budget, capture=True):
+ self.assertEqual(cwd, self.project)
+ self.assertEqual(len(list((self.home / "init.d").glob("dbx-ci-acquire-*.gradle"))), 1)
+ self.calls.append((command, dict(env), budget, capture))
+ result = next(responses)
+ if isinstance(result, BaseException):
+ raise result
+ return result
+
+ with patch.dict(os.environ, {"GRADLE_USER_HOME": str(self.home)}):
+ try:
+ return gradle.acquire_and_run(self.project, "agents", ["original", "test", "shadowJar", "--continue"],
+ runner=run, sleep=self.sleeps.append)
+ finally:
+ self.assertFalse(list((self.home / "init.d").glob("dbx-ci-acquire-*.gradle")))
+
+ def test_success_acquires_before_one_offline_original_command(self):
+ self.assertEqual(self.execute([OK, OK, OK]), 0)
+ self.assertEqual([call[2] for call in self.calls], [180, 600, None])
+ self.assertEqual(self.calls[1][0][-1], "dbxCiAcquireDependencies")
+ self.assertEqual(self.calls[2][0], ["original", "test", "shadowJar", "--continue"])
+ self.assertEqual(self.calls[2][1]["DBX_CI_GRADLE_PHASE"], "execute")
+ self.assertEqual(self.calls[2][1]["DBX_CI_GRADLE_SOURCE"], "primary")
+ self.assertFalse(self.calls[2][3])
+ self.assertEqual(self.sleeps, [])
+
+ def test_public_forbidden_switches_once_without_repeating_original(self):
+ self.assertEqual(self.execute([OK, failure(), OK, OK]), 0)
+ self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls],
+ ["primary", "primary", "backup", "backup"])
+ self.assertEqual(self.sleeps, [])
+
+ def test_transient_retries_once_before_backup(self):
+ self.assertEqual(self.execute([OK, failure(503), failure(503), OK, OK]), 0)
+ self.assertEqual(self.sleeps, [2])
+ self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls[1:]],
+ ["primary", "primary", "backup", "backup"])
+
+ def test_backup_failure_and_execution_failure_are_terminal(self):
+ for responses, count in [([OK, failure(), failure(), failure()], 4),
+ ([OK, OK, failure()], 3)]:
+ with self.subTest(responses=responses):
+ self.calls.clear()
+ self.assertEqual(self.execute(responses), 1)
+ self.assertEqual(len(self.calls), count)
+
+ def test_forbidden_preserves_remaining_attempt_for_backup(self):
+ backup_error = failure(503, "https://maven-central.storage-download.googleapis.com/maven2/example/a.pom")
+ self.assertEqual(self.execute([OK, failure(), backup_error, OK, OK]), 0)
+ self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls[1:]],
+ ["primary", "backup", "backup", "backup"])
+ self.assertEqual(self.sleeps, [2])
+
+ def test_timeout_recovers_only_acquisition(self):
+ timeout = (-9, "Downloading dependencies\n", True)
+ self.assertEqual(self.execute([OK, timeout, timeout, OK, OK]), 0)
+ self.assertEqual(self.sleeps, [2])
+
+ def test_wrapper_has_separate_bounded_retry_and_no_project_task(self):
+ self.assertEqual(self.execute([(1, "java.net.SocketTimeoutException: Read timed out", False), OK, OK, OK]), 0)
+ self.assertEqual(self.sleeps, [2])
+ self.assertTrue(all(call[0][-1] == "--version" for call in self.calls[:2]))
+
+ def test_deterministic_private_and_mixed_failures_do_not_switch(self):
+ errors = [failure(404), failure(401), failure(url="https://private.test/maven2/a.pom"),
+ failure(url="https://user:[email protected]/maven2/a.pom"),
+ failure(url="https://repo.maven.apache.org/maven2/a.pom?token=secret"),
+ (1, failure()[1] + "\n> No space left on device", False),
+ (1, failure()[1].replace("* Try:", "> PKIX certificate failure\n* Try:"), False),
+ (1, failure()[1].replace("* Try:", "> An unknown failure\n* Try:"), False)]
+ for error in errors:
+ with self.subTest(error=error):
+ self.calls.clear()
+ self.assertEqual(self.execute([OK, error]), 1)
+ self.assertEqual(len(self.calls), 2)
+
+ def test_unknown_wrapper_failure_does_not_enter_acquisition(self):
+ self.assertEqual(self.execute([(1, "checksum verification failed", False)]), 1)
+ self.assertEqual(len(self.calls), 1)
+
+ def test_cancellation_never_switches_and_preserves_other_init_scripts(self):
+ directory = self.home / "init.d"
+ directory.mkdir(parents=True)
+ other = directory / "existing.gradle"
+ other.write_text("// existing")
+ with self.assertRaises(gradle.Cancelled):
+ self.execute([OK, gradle.Cancelled(signal.SIGTERM)])
+ self.assertEqual(len(self.calls), 2)
+ self.assertEqual(other.read_text(), "// existing")
+
+ def test_original_exception_still_cleans_up(self):
+ with self.assertRaises(OSError):
+ self.execute([OK, OK, OSError("cannot start original")])
+ self.assertEqual(len(self.calls), 3)
+
+ @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups")
+ def test_spawn_cancellation_reaps_child_before_init_cleanup(self):
+ events = []
+ with patch.object(gradle.signal, "signal"), patch.object(gradle.os, "killpg", side_effect=lambda *_: events.append("kill")), \
+ patch.object(gradle.subprocess, "Popen") as popen:
+ child = popen.return_value
+ child.communicate.side_effect = lambda: (events.append("reap") or "", None)
+
+ def spawn(*_args, **_kwargs):
+ gradle.cancel(signal.SIGTERM, None)
+ return child
+
+ popen.side_effect = spawn
+ with self.assertRaises(gradle.Cancelled):
+ gradle.run_process(["gradle"], self.project, {}, 1)
+ self.assertEqual(events, ["kill", "reap"])
+
+ @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+ def test_timeout_and_cancellation_dispose_owned_descendants(self):
+ for signum in [0, signal.SIGTERM, signal.SIGINT]:
+ with self.subTest(signum=signum):
+ case = self.root / str(signum)
+ case.mkdir()
+ pids = case / "pids"
+ harness = textwrap.dedent("""
+ import importlib.util, os, signal, sys
+ from pathlib import Path
+ spec = importlib.util.spec_from_file_location('acquire', sys.argv[1])
+ acquire = importlib.util.module_from_spec(spec); spec.loader.exec_module(acquire)
+ signal.signal(signal.SIGINT, acquire.cancel)
+ signal.signal(signal.SIGTERM, acquire.cancel)
+ os.environ['GRADLE_USER_HOME'] = sys.argv[2]
+ payload = '''import os, subprocess, sys, time
+ child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])
+ with open(sys.argv[1], 'a') as output: print(os.getpid(), child.pid, file=output)
+ time.sleep(60)
+ '''
+ def run(command, cwd, env, budget, capture=True):
+ if command[-1] == '--version': return 0, '', False
+ if not capture: raise RuntimeError('execution reached without acquisition')
+ return acquire.run_process([sys.executable, '-c', payload, sys.argv[3]], cwd, env, 1)
+ try:
+ code = acquire.acquire_and_run(Path(sys.argv[2]), 'agents', ['original'], runner=run, sleep=lambda _: None)
+ sys.exit(code)
+ except acquire.Cancelled as error:
+ sys.exit(128 + error.signum)
+ """)
+ process = subprocess.Popen([sys.executable, "-c", harness, gradle.__file__,
+ str(case), str(pids)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ try:
+ deadline = time.monotonic() + 10
+ while (not pids.exists() or not pids.read_text().strip()) and process.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertTrue(pids.exists() and pids.read_text().strip(), "fake acquisition did not start")
+ if signum:
+ process.send_signal(signum)
+ output = process.communicate(timeout=10)[0]
+ self.assertNotEqual(process.returncode, 0, output)
+ self.assertNotIn("phase=execute", output)
+ self.assertEqual(len(pids.read_text().splitlines()), 1 if signum else 3)
+ self.assertFalse(list((case / "init.d").glob("dbx-ci-acquire-*.gradle")))
+ for pid in pids.read_text().split():
+ status = Path(f"/proc/{pid}/stat")
+ deadline = time.monotonic() + 5
+ while status.exists() and status.read_text().split()[2] != "Z" and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertTrue(not status.exists() or status.read_text().split()[2] == "Z", f"owned child {pid} survived")
+ finally:
+ if pids.exists():
+ for line in pids.read_text().splitlines():
+ try:
+ os.killpg(int(line.split()[0]), signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ if process.poll() is None:
+ process.kill()
+ process.communicate(timeout=5)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/ci-install-mold.sh b/.github/scripts/ci-install-mold.sh
new file mode 100644
index 0000000000..b3a0b2593a
--- /dev/null
+++ b/.github/scripts/ci-install-mold.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+archive=$(mktemp)
+trap 'rm -f -- "$archive"' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+curl --fail --location --silent --show-error \
+ --retry 2 --retry-connrefused --retry-delay 2 \
+ --connect-timeout 15 --max-time 120 --retry-max-time 360 \
+ https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
+ --output "$archive"
+sudo tar -C /usr/local --strip-components=1 -xzf "$archive"
+mold --version
diff --git a/.github/scripts/ci-install-mold.test.py b/.github/scripts/ci-install-mold.test.py
new file mode 100644
index 0000000000..c47f21a7d3
--- /dev/null
+++ b/.github/scripts/ci-install-mold.test.py
@@ -0,0 +1,117 @@
+import http.server
+import os
+from pathlib import Path
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import unittest
+
+SCRIPT = Path(__file__).with_name("ci-install-mold.sh")
+
+
[email protected](sys.platform == "linux", "requires the Linux CI shell and curl")
+class MoldDownloadTests(unittest.TestCase):
+ def setUp(self):
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.root = Path(self.directory.name)
+ self.bin = self.root / "bin"
+ self.bin.mkdir()
+ self.downloads = self.root / "downloads"
+ self.downloads.mkdir()
+ self.statuses = [200]
+ self.requests = 0
+ self.started = threading.Event()
+ self.release = threading.Event()
+ owner = self
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ status = owner.statuses[min(owner.requests, len(owner.statuses) - 1)]
+ owner.requests += 1
+ self.send_response(200 if status == "partial" else status)
+ self.send_header("Content-Length", "1000" if status == "partial" else "7")
+ self.end_headers()
+ self.wfile.write(b"archive")
+ self.wfile.flush()
+ owner.started.set()
+ if status == "partial":
+ owner.release.wait(10)
+
+ def log_message(self, *_args):
+ pass
+
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.addCleanup(server.server_close)
+ self.addCleanup(server.shutdown)
+ self.addCleanup(self.release.set)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ self.env = {**os.environ, "PATH": str(self.bin) + os.pathsep + os.environ["PATH"],
+ "REAL_CURL": shutil.which("curl"), "TEST_URL": f"http://127.0.0.1:{server.server_port}/mold",
+ "TEST_ROOT": str(self.root), "TMPDIR": str(self.downloads)}
+ self.tool("curl", """import os, sys
+args = sys.argv[1:]
+assert '--retry-all-errors' not in args
+for option, value in [('--retry', '2'), ('--connect-timeout', '15'), ('--max-time', '120'), ('--retry-max-time', '360')]:
+ assert args[args.index(option) + 1] == value
+assert 'https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz' in args
+args = [os.environ['TEST_URL'] if arg.startswith('https://') else arg for arg in args]
+os.execv(os.environ['REAL_CURL'], [os.environ['REAL_CURL'], *args])
+""")
+ self.tool("sudo", """import os, sys
+from pathlib import Path
+assert sys.argv[1:6] == ['tar', '-C', '/usr/local', '--strip-components=1', '-xzf']
+assert Path(sys.argv[-1]).read_bytes() == b'archive'
+with (Path(os.environ['TEST_ROOT']) / 'installed').open('a') as output: output.write('once\\n')
+""")
+ self.tool("mold", "print('mold 2.42.1')\n")
+
+ def tool(self, name, body):
+ path = self.bin / name
+ path.write_text(f"#!{sys.executable}\n" + body)
+ path.chmod(0o755)
+
+ def run_script(self):
+ result = subprocess.run(["bash", str(SCRIPT)], env=self.env, capture_output=True, text=True, timeout=20)
+ self.assertEqual(list(self.downloads.iterdir()), [])
+ return result
+
+ def test_transient_status_recovers_before_one_install(self):
+ self.statuses = [503, 503, 200]
+ result = self.run_script()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(self.requests, 3)
+ self.assertEqual((self.root / "installed").read_text(), "once\n")
+
+ def test_not_found_and_exhausted_retries_never_install(self):
+ for statuses, requests in [([404], 1), ([503], 3)]:
+ with self.subTest(statuses=statuses):
+ self.statuses = statuses
+ self.requests = 0
+ result = self.run_script()
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.requests, requests)
+ self.assertFalse((self.root / "installed").exists())
+
+ def test_cancellation_cleans_partial_file_without_install(self):
+ self.statuses = ["partial"]
+ process = subprocess.Popen(["bash", str(SCRIPT)], env=self.env, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, text=True, start_new_session=True)
+ try:
+ self.assertTrue(self.started.wait(5), "curl did not start")
+ os.killpg(process.pid, signal.SIGTERM)
+ process.communicate(timeout=5)
+ self.assertNotEqual(process.returncode, 0)
+ self.assertFalse((self.root / "installed").exists())
+ self.assertEqual(list(self.downloads.iterdir()), [])
+ finally:
+ if process.poll() is None:
+ os.killpg(process.pid, signal.SIGKILL)
+ process.communicate(timeout=5)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/ci-install-system-deps.py b/.github/scripts/ci-install-system-deps.py
new file mode 100644
index 0000000000..e5c5ff087f
--- /dev/null
+++ b/.github/scripts/ci-install-system-deps.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Download Ubuntu dependencies with at most two mirror switches, then install offline."""
+
+import argparse
+import os
+import re
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+SOURCES = Path("/etc/apt/sources.list.d/ubuntu.sources")
+PRIMARY = "https://archive.ubuntu.com/ubuntu/"
+BACKUP = "https://mirrors.edge.kernel.org/ubuntu/"
+LAST_RESORT = "http://azure.archive.ubuntu.com/ubuntu/"
+MIRRORS = [("primary", PRIMARY), ("backup", BACKUP), ("last-resort", LAST_RESORT)]
+PACKAGES = ["libwebkit2gtk-4.1-dev", "libappindicator3-dev", "librsvg2-dev",
+ "patchelf", "libssl-dev", "libsecret-1-dev"]
+PACKAGE_GROUPS = {"gtk": PACKAGES, "packaging": ["zstd"],
+ "duckdb": ["libdbus-1-dev", "pkg-config"]}
+ACQUIRE_OPTIONS = ["-o", "Acquire::Retries=0", "-o", "Acquire::http::Timeout=30",
+ "-o", "Acquire::https::Timeout=30", "-o", "APT::Update::Error-Mode=any"]
+starting_process = False
+pending_cancel = None
+
+
+def rewrite_ubuntu_uris(source, mirror):
+ known = {"mirror+file:/etc/apt/apt-mirrors.txt", PRIMARY, PRIMARY.rstrip("/"),
+ BACKUP, BACKUP.rstrip("/"), LAST_RESORT, LAST_RESORT.rstrip("/")}
+ found = 0
+ in_uris = False
+ result = []
+
+ def replace(match):
+ nonlocal found
+ if match.group() in known:
+ found += 1
+ return mirror
+ return match.group()
+
+ for line in source.splitlines(keepends=True):
+ if line.lstrip().startswith("#"):
+ result.append(line)
+ continue
+ field = re.match(r"^([A-Za-z][A-Za-z0-9-]*):", line)
+ if field:
+ in_uris = field[1].lower() == "uris"
+ elif not line.strip() or not line[0].isspace():
+ in_uris = False
+ if in_uris:
+ prefix = field.end() if field else 0
+ line = line[:prefix] + re.sub(r"\S+", replace, line[prefix:])
+ result.append(line)
+ if not found:
+ raise ValueError("Ubuntu sources do not contain the expected runner mirror URI")
+ return "".join(result)
+
+
+def is_retryable_acquisition_failure(output, timed_out):
+ if re.search(r"signature|NO_PUBKEY|GPG error|certificate|not signed|Release file|"
+ r"Hash Sum mismatch|unexpected size|Unable to locate package|unmet dependencies|"
+ r"held broken packages|Could not get lock|Unable to acquire|Waiting for cache lock|"
+ r"dpkg was interrupted|Permission denied|No space left|404\s", output, re.I):
+ return False
+ reasons = []
+ for line in output.splitlines():
+ fetch = re.match(r"[EW]: Failed to fetch \S+\s+(.+)", line)
+ if fetch:
+ reasons.append(fetch[1])
+ elif line.startswith("E:") and not line.startswith((
+ "E: Unable to fetch some archives,", "E: Some index files failed to download."
+ )):
+ return False
+ network = re.compile(r"Temporary failure resolving|Could not resolve|Could not connect|"
+ r"Connection (?:failed|timed out|reset)|Network is unreachable|"
+ r"Error reading from server|Remote end closed|"
+ r"(?:^|\s)(?:429|500|502|503|504)(?:\s|$)", re.I)
+ if any(not network.search(reason) for reason in reasons):
+ return False
+ return timed_out or bool(reasons)
+
+
+class Cancelled(BaseException):
+ def __init__(self, signum):
+ self.signum = signum
+
+
+def cancel(signum, _frame):
+ global pending_cancel
+ # Finish child disposal and source restoration even if cancellation repeats.
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ if starting_process:
+ pending_cancel = signum
+ return
+ raise Cancelled(signum)
+
+
+def dispose(process):
+ try:
+ os.killpg(process.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ return process.communicate()[0]
+
+
+def run_apt(arguments, budget):
+ global starting_process, pending_cancel
+ started = time.monotonic()
+ process = None
+ timed_out = False
+ try:
+ starting_process = True
+ process = subprocess.Popen(
+ ["apt-get", *ACQUIRE_OPTIONS, *arguments], stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, text=True, env={**os.environ, "LC_ALL": "C"},
+ start_new_session=True,
+ )
+ starting_process = False
+ if pending_cancel is not None:
+ signum, pending_cancel = pending_cancel, None
+ raise Cancelled(signum)
+ try:
+ output = process.communicate(timeout=budget)[0]
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ output = dispose(process)
+ except BaseException:
+ if process is not None:
+ dispose(process)
+ raise
+ finally:
+ starting_process = False
+ print(output, end="", flush=True)
+ print(f"APT finished elapsed={time.monotonic() - started:.1f}s "
+ f"exit={process.returncode} timed_out={timed_out}", flush=True)
+ return process.returncode, output, timed_out
+
+
+def install(source_path=SOURCES, group="gtk"):
+ packages = PACKAGE_GROUPS[group]
+ original = source_path.read_bytes()
+ source = original.decode("utf-8")
+ candidates = [(label, rewrite_ubuntu_uris(source, mirror)) for label, mirror in MIRRORS]
+ try:
+ for index, (label, candidate) in enumerate(candidates):
+ source_path.write_bytes(candidate.encode("utf-8"))
+ for phase, arguments, budget in [
+ ("index", ["update"], 60),
+ ("download", ["--download-only", "install", "-y", *packages], 120),
+ ]:
+ print(f"APT source={label} phase={phase} budget={budget}s", flush=True)
+ code, output, timed_out = run_apt(arguments, budget)
+ if code != 0 or timed_out:
+ if index + 1 < len(candidates) and is_retryable_acquisition_failure(output, timed_out):
+ reason = "acquisition deadline" if timed_out else "transient acquisition error"
+ print(f"APT switching to next source {MIRRORS[index + 1][1]}: {reason}", flush=True)
+ break
+ raise RuntimeError(f"APT {label} {phase} failed (exit={code}, timed_out={timed_out})")
+ else:
+ break
+ print("APT phase=install budget=1200s downloads=disabled", flush=True)
+ code, _output, timed_out = run_apt(["--no-download", "install", "-y", *packages], 1200)
+ if code != 0 or timed_out:
+ raise RuntimeError(f"APT offline install failed (exit={code}, timed_out={timed_out})")
+ finally:
+ source_path.write_bytes(original)
+
+
+if __name__ == "__main__":
+ signal.signal(signal.SIGINT, cancel)
+ signal.signal(signal.SIGTERM, cancel)
+ try:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--group", choices=PACKAGE_GROUPS, default="gtk")
+ install(group=parser.parse_args().group)
+ except Cancelled as error:
+ print("APT cancelled; no mirror retry", file=sys.stderr)
+ sys.exit(128 + error.signum)
+ except Exception as error:
+ print(f"::error::{error}", file=sys.stderr)
+ sys.exit(1)
diff --git a/.github/scripts/ci-install-system-deps.test.py b/.github/scripts/ci-install-system-deps.test.py
new file mode 100644
index 0000000000..19e2e6ee30
--- /dev/null
+++ b/.github/scripts/ci-install-system-deps.test.py
@@ -0,0 +1,282 @@
+import importlib.util
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+SCRIPT = Path(__file__).with_name("ci-install-system-deps.py")
+spec = importlib.util.spec_from_file_location("deps", SCRIPT)
+deps = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(deps)
+
+SOURCE = """Types: deb
+URIs: mirror+file:/etc/apt/apt-mirrors.txt
+Suites: noble noble-updates noble-backports
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+
+Types: deb
+URIs:
+ mirror+file:/etc/apt/apt-mirrors.txt
+Suites: noble-security
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+
+Types: deb
+URIs: https://security.ubuntu.com/ubuntu/
+Suites: noble-security
+Components: main
+
+Types: deb
+URIs: https://example.test/apt
+Suites: stable
+Components: main
+"""
+OK = (0, "", False)
+TIMEOUT = (-9, "", True)
+NETWORK = (100, "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb Connection timed out\n"
+ "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n", False)
+
+
+class InstallerTests(unittest.TestCase):
+ def setUp(self):
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.source = Path(self.directory.name) / "ubuntu.sources"
+ self.source.write_bytes(SOURCE.encode())
+ self.calls = []
+
+ def execute(self, results, group="gtk"):
+ responses = iter(results)
+
+ def run(arguments, budget):
+ self.calls.append((arguments, budget, self.source.read_text()))
+ return next(responses)
+
+ with patch.object(deps, "run_apt", side_effect=run):
+ try:
+ deps.install(self.source, group)
+ finally:
+ self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+
+ def test_success_downloads_before_one_offline_install(self):
+ self.execute([OK, OK, OK])
+ self.assertEqual([call[1] for call in self.calls], [60, 120, 1200])
+ self.assertEqual(self.calls[1][0], ["--download-only", "install", "-y", *deps.PACKAGES])
+ self.assertEqual(self.calls[2][0], ["--no-download", "install", "-y", *deps.PACKAGES])
+ self.assertTrue(all(deps.BACKUP not in call[2] for call in self.calls))
+
+ def test_fixed_package_groups_preserve_each_caller_list(self):
+ for group, packages in {"packaging": ["zstd"],
+ "duckdb": ["libdbus-1-dev", "pkg-config"]}.items():
+ with self.subTest(group=group):
+ self.calls.clear()
+ self.execute([OK, OK, OK], group)
+ self.assertEqual(self.calls[1][0], ["--download-only", "install", "-y", *packages])
+ self.assertEqual(self.calls[2][0], ["--no-download", "install", "-y", *packages])
+
+ def test_official_primary_and_known_runner_sources(self):
+ self.assertEqual(deps.PRIMARY, "https://archive.ubuntu.com/ubuntu/")
+ for uri in [deps.PRIMARY, deps.BACKUP, deps.LAST_RESORT, "mirror+file:/etc/apt/apt-mirrors.txt"]:
+ with self.subTest(uri=uri):
+ source = SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", uri)
+ rewritten = deps.rewrite_ubuntu_uris(source, deps.PRIMARY)
+ self.assertIn("URIs: https://archive.ubuntu.com/ubuntu/", rewritten)
+ self.assertIn("URIs: https://security.ubuntu.com/ubuntu/", rewritten)
+ self.assertIn("Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg", rewritten)
+
+ def test_timeout_or_network_failure_switches_once_and_reuses_cache(self):
+ for failure in [TIMEOUT, NETWORK]:
+ with self.subTest(failure=failure):
+ self.calls.clear()
+ self.execute([OK, failure, OK, OK, OK])
+ self.assertEqual([call[1] for call in self.calls], [60, 120, 60, 120, 1200])
+ self.assertIn(deps.PRIMARY, self.calls[1][2])
+ self.assertNotIn(deps.PRIMARY, self.calls[2][2])
+ self.assertIn(deps.BACKUP, self.calls[2][2])
+ self.assertFalse(any("clean" in call[0] or "--fix-missing" in call[0] for call in self.calls))
+ self.assertTrue(all(not any("Dir::Cache" in arg for arg in call[0]) for call in self.calls))
+
+ def test_network_index_failure_refreshes_backup_before_downloading(self):
+ self.execute([NETWORK, OK, OK, OK])
+ self.assertEqual([call[0][0] for call in self.calls], ["update", "update", "--download-only", "--no-download"])
+
+ def test_all_three_sources_fail_without_installing_partial_downloads(self):
+ with self.assertRaises(RuntimeError):
+ self.execute([OK, TIMEOUT, OK, NETWORK, OK, NETWORK])
+ self.assertEqual(len(self.calls), 6)
+ self.assertFalse(any("--no-download" in call[0] for call in self.calls))
+
+ def test_second_switch_is_last_resort_and_never_loops(self):
+ self.execute([OK, TIMEOUT, OK, NETWORK, OK, OK, OK])
+ self.assertEqual(len(self.calls), 7)
+ self.assertIn(deps.PRIMARY, self.calls[1][2])
+ self.assertIn(deps.BACKUP, self.calls[3][2])
+ self.assertIn(deps.LAST_RESORT, self.calls[5][2])
+ self.assertEqual(self.calls[6][0][0], "--no-download")
+
+ def test_deterministic_and_unknown_errors_never_switch(self):
+ for output in ["E: Unable to locate package missing", "W: GPG error: invalid signatures",
+ "E: Failed to fetch https://archive.ubuntu.com/a.deb Certificate verification failed",
+ "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb 404 Not Found",
+ "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb Hash Sum mismatch",
+ "E: Could not get lock /var/lib/dpkg/lock", "E: No space left on device",
+ "E: Unknown error", NETWORK[1] + "E: unmet dependencies"]:
+ for timed_out in [False, True]:
+ with self.subTest(output=output, timed_out=timed_out):
+ self.calls.clear()
+ with self.assertRaises(RuntimeError):
+ self.execute([OK, (100, output, timed_out)])
+ self.assertEqual(len(self.calls), 2)
+ self.assertFalse(deps.is_retryable_acquisition_failure("", False))
+ self.assertFalse(deps.is_retryable_acquisition_failure(
+ NETWORK[1] + "E: Failed to fetch https://example.test/a.deb Unknown error", False))
+
+ def test_http_transient_errors_are_allowed_but_not_unknown_fetch_errors(self):
+ for status in [429, 500, 502, 503, 504]:
+ self.assertTrue(deps.is_retryable_acquisition_failure(
+ f"E: Failed to fetch https://archive.ubuntu.com/a.deb {status} Server error", False))
+ self.assertFalse(deps.is_retryable_acquisition_failure(
+ "E: Failed to fetch https://archive.ubuntu.com/a.deb Something went wrong", False))
+
+ def test_install_failure_is_terminal_without_retry(self):
+ for failure in [TIMEOUT, NETWORK, (100, "E: dpkg failed", False)]:
+ self.calls.clear()
+ with self.assertRaises(RuntimeError):
+ self.execute([OK, OK, failure])
+ self.assertEqual(len(self.calls), 3)
+
+ def test_source_rewrite_preserves_all_non_uri_fields_and_security(self):
+ rewritten = deps.rewrite_ubuntu_uris(SOURCE, deps.BACKUP)
+ self.assertEqual(rewritten, SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", deps.BACKUP))
+ self.assertEqual(deps.rewrite_ubuntu_uris(SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", deps.PRIMARY), deps.BACKUP), rewritten)
+ self.source.write_text("Types: deb\nURIs: https://unknown.test/ubuntu\nSuites: noble\n")
+ original = self.source.read_bytes()
+ with self.assertRaises(ValueError), patch.object(deps, "run_apt") as run:
+ deps.install(self.source)
+ run.assert_not_called()
+ self.assertEqual(self.source.read_bytes(), original)
+
+ def test_run_apt_uses_an_owned_process_group_and_c_locale(self):
+ with patch.object(deps.subprocess, "Popen") as popen:
+ popen.return_value.communicate.return_value = ("downloaded\n", None)
+ popen.return_value.returncode = 0
+ self.assertEqual(deps.run_apt(["--download-only", "install", "-y", *deps.PACKAGES], 120), (0, "downloaded\n", False))
+ self.assertTrue(popen.call_args.kwargs["start_new_session"])
+ self.assertEqual(popen.call_args.kwargs["env"]["LC_ALL"], "C")
+ self.assertIn("Acquire::Retries=0", popen.call_args.args[0])
+
+ @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups")
+ def test_cancellation_during_spawn_reaps_before_restoring_sources(self):
+ events = []
+ write = Path.write_bytes
+
+ def record_write(path, content):
+ if content == SOURCE.encode():
+ events.append("restore")
+ return write(path, content)
+
+ with patch.object(deps.subprocess, "Popen") as popen, \
+ patch.object(deps.signal, "signal"), \
+ patch.object(deps.os, "killpg", side_effect=lambda *_: events.append("kill")), \
+ patch.object(Path, "write_bytes", autospec=True, side_effect=record_write):
+ process = popen.return_value
+ process.communicate.side_effect = lambda: (events.append("reap") or "", None)
+
+ def spawn(*_args, **_kwargs):
+ events.append("spawn")
+ deps.cancel(signal.SIGTERM, None)
+ return process
+
+ popen.side_effect = spawn
+ with self.assertRaises(deps.Cancelled):
+ deps.install(self.source)
+ popen.assert_called_once()
+ self.assertEqual(events, ["spawn", "kill", "reap", "restore"])
+ self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+
+ def run_child_case(self, signum=0):
+ pid_file = Path(self.directory.name) / f"child-{signum}.pid"
+ payload = textwrap.dedent("""
+ import os, subprocess, sys, time
+ child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])
+ open(sys.argv[1], 'w').write(f'{os.getpid()} {child.pid}')
+ time.sleep(60)
+ """)
+ harness = textwrap.dedent("""
+ import importlib.util, signal, subprocess, sys
+ from pathlib import Path
+ spec = importlib.util.spec_from_file_location('deps', sys.argv[1])
+ deps = importlib.util.module_from_spec(spec); spec.loader.exec_module(deps)
+ deps.signal.signal(signal.SIGINT, deps.cancel)
+ deps.signal.signal(signal.SIGTERM, deps.cancel)
+ original_popen = subprocess.Popen
+ deps.subprocess.Popen = lambda *a, **kw: original_popen([sys.executable, '-c', sys.argv[3], sys.argv[4]], **kw)
+ original_run = deps.run_apt
+ deps.run_apt = lambda args, budget: original_run(args, 3)
+ try:
+ deps.install(Path(sys.argv[2]))
+ except deps.Cancelled as error:
+ sys.exit(128 + error.signum)
+ except RuntimeError:
+ sys.exit(1)
+ """)
+ process = subprocess.Popen([sys.executable, "-c", harness, str(SCRIPT), str(self.source), payload, str(pid_file)],
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+ def cleanup():
+ if pid_file.exists() and pid_file.read_text():
+ try:
+ os.killpg(int(pid_file.read_text().split()[0]), signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ if process.poll() is None:
+ process.kill()
+ process.communicate(timeout=5)
+
+ self.addCleanup(cleanup)
+ deadline = time.monotonic() + 10
+ while not pid_file.exists() and process.poll() is None and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertTrue(pid_file.exists(), "fake apt did not start")
+ if signum:
+ process.send_signal(signum)
+ output = process.communicate(timeout=15)[0]
+ self.assertEqual(process.returncode, 128 + signum if signum else 1, output)
+ self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+ self.assertNotIn("phase=install", output)
+ child = Path(f"/proc/{pid_file.read_text().split()[1]}/stat")
+
+ def child_running():
+ try:
+ return child.read_text().split()[2] != "Z"
+ except FileNotFoundError:
+ return False
+
+ deadline = time.monotonic() + 5
+ while child_running() and time.monotonic() < deadline:
+ time.sleep(0.02)
+ self.assertFalse(child_running(), "descendant still running")
+ return output
+
+ @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+ def test_timeout_disposes_descendants_before_backup(self):
+ output = self.run_child_case()
+ self.assertIn("timed_out=True", output)
+ self.assertEqual(output.count("switching to next source"), 2)
+
+ @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+ def test_cancellation_disposes_descendants_without_backup_and_restores_sources(self):
+ for signum in [signal.SIGINT, signal.SIGTERM]:
+ with self.subTest(signum=signum):
+ output = self.run_child_case(signum)
+ self.assertNotIn("switching to next source", output)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/ci-plan.test.mjs b/.github/scripts/ci-plan.test.mjs
index c72810fcb0..47af7321dc 100644
--- a/.github/scripts/ci-plan.test.mjs
+++ b/.github/scripts/ci-plan.test.mjs
@@ -232,7 +232,21 @@ test("code generation and test-only dependencies participate in impact analysis"
assert.ok(plan(["crates/dbx-formats/src/lib.rs"], { metadata: fixture }).affected_packages.includes("dbx-types"));
});
-for (const file of ["Cargo.toml", "Cargo.lock", ".cargo/config.toml", "rust-toolchain.toml", "vendor/wry/src/lib.rs", "crates/dbx-sql/Cargo.toml", ".github/workflows/ci.yml", ".github/scripts/ci-plan.mjs", ".github/scripts/ci-gate.test.mjs"]) {
+for (const file of [
+ "Cargo.toml",
+ "Cargo.lock",
+ ".cargo/config.toml",
+ "rust-toolchain.toml",
+ "vendor/wry/src/lib.rs",
+ "crates/dbx-sql/Cargo.toml",
+ ".github/workflows/ci.yml",
+ ".github/scripts/ci-plan.mjs",
+ ".github/scripts/ci-gate.test.mjs",
+ ".github/scripts/ci-gradle.py",
+ ".github/scripts/ci-gradle.init.gradle",
+ ".github/scripts/ci-install-system-deps.py",
+ ".github/scripts/ci-install-mold.sh",
+]) {
test(`shared input ${file} retains full workspace and Agent coverage`, () => {
const result = plan([file]);
assert.deepEqual(groups(result), ["workspace"]);
diff --git a/.github/scripts/ci-workflow.test.mjs b/.github/scripts/ci-workflow.test.mjs
index 6f162e058f..df66b89898 100644
--- a/.github/scripts/ci-workflow.test.mjs
+++ b/.github/scripts/ci-workflow.test.mjs
@@ -279,15 +279,36 @@ test("Rust build timings remain available after failures without waiting for can
assert.ok(job("agent-rust").includes('--manifest-path "drivers/$DRIVER/Cargo.toml" --locked --no-fail-fast --timings'));
});
-test("Linux Rust dependency installs bound apt mirror stalls", () => {
- for (const name of ["packages", "rust-fmt-clippy", "rust-test", "agent-rust"]) {
- const content = job(name);
- const deadlines = [...content.matchAll(/sudo timeout --kill-after=30s (\d+)m apt-get/g)].map((match) => Number(match[1]));
- assert.deepEqual(deadlines, [10, name === "agent-rust" ? 10 : 20]);
- for (const option of ["Acquire::Retries=3", "Acquire::http::Timeout=30", "Acquire::https::Timeout=30"]) {
- assert.ok(content.includes(option));
- }
+test("Linux dependency installs share bounded acquisition before offline installation", () => {
+ for (const name of ["packages", "rust-fmt-clippy", "rust-test"]) {
+ assert.ok(job(name).includes("sudo python3 .github/scripts/ci-install-system-deps.py"));
+ assert.ok(!job(name).includes("apt-get"));
}
+ for (const name of ["agent-checks", "agent-java"]) {
+ assert.ok(job(name).includes("sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging"));
+ }
+ assert.ok(job("agent-rust").includes("sudo python3 ../.github/scripts/ci-install-system-deps.py --group duckdb"));
+ assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-install-system-deps.test.py"));
+});
+
+test("Java CI acquires before original commands without retrying packaging or tests", () => {
+ assert.ok(job("agent-java").includes("python3 ../.github/scripts/ci-gradle.py agents -- ./gradlew test shadowJar --continue"));
+ for (const name of ["jdbc", "offline-jdbc-release"]) {
+ assert.ok(job(name).includes("python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh"));
+ }
+ assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-gradle.test.py"));
+});
+
+test("mold downloads are isolated from extraction in CI and keepalive", () => {
+ for (const name of ["packages", "rust-test"]) {
+ assert.ok(job(name).includes("run: bash .github/scripts/ci-install-mold.sh"));
+ }
+ assert.ok(job("agent-rust").includes("run: bash ../.github/scripts/ci-install-mold.sh"));
+ const keepalive = readFileSync(new URL("../workflows/cache-keepalive.yml", import.meta.url), "utf8");
+ assert.ok(keepalive.includes("run: bash .github/scripts/ci-install-mold.sh"));
+ assert.ok(keepalive.includes("sudo python3 .github/scripts/ci-install-system-deps.py"));
+ assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-install-mold.test.py"));
+ assert.ok(job("github-scripts").includes("./.github/scripts/prepare-webview2-win7-runtime.test.ps1"));
});
for (const name of ["rust-test", "agent-integration"]) {
@@ -297,7 +318,14 @@ for (const name of ["rust-test", "agent-integration"]) {
const block = step.slice(step.indexOf(" timeout 30s")).replace(/^ /gm, "");
const directory = mkdtempSync(path.join(tmpdir(), "dbx-docker-ready-"));
try {
- const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", `
+ const result = spawnSync(
+ "bash",
+ [
+ "-e",
+ "-o",
+ "pipefail",
+ "-c",
+ `
sudo() { "$@"; }
systemctl() { printf 'restart\\n' >> "$STATE/restarts"; }
timeout() { shift; "$@"; }
@@ -314,7 +342,10 @@ for (const name of ["rust-test", "agent-integration"]) {
}
${block}
printf 'verified\\n'
- `], { env: { ...process.env, STATE: directory.replaceAll("\\", "/"), SCENARIO: scenario }, encoding: "utf8", timeout: 15000 });
+ `,
+ ],
+ { env: { ...process.env, STATE: directory.replaceAll("\\", "/"), SCENARIO: scenario }, encoding: "utf8", timeout: 15000 },
+ );
assert.ifError(result.error);
assert.equal(readFileSync(path.join(directory, "restarts"), "utf8"), "restart\n");
const count = Number(readFileSync(path.join(directory, "count"), "utf8"));
@@ -389,7 +420,8 @@ test("Windows compatibility jobs cache Rust compilation without wrapping C or C+
const win7 = job("windows-win7-bundle");
assert.ok(win7.includes("needs.changes.outputs.windows_win7_bundle == 'true'"));
assert.doesNotMatch(win7, /x86_64-pc-windows-msvc|Setup Rust for standard Windows/);
- for (const setting of ["RUSTC_WRAPPER: sccache", "SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}", "SCCACHE_GHA_VERSION: win7-webview2-1.0.902.49-v1", 'SCCACHE_IDLE_TIMEOUT: "0"', 'CARGO_PROFILE_RELEASE_LTO: "off"', 'CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "8"']) assert.ok(win7.includes(setting));
+ for (const setting of ["RUSTC_WRAPPER: sccache", "SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}", "SCCACHE_GHA_VERSION: win7-webview2-1.0.902.49-v1", 'SCCACHE_IDLE_TIMEOUT: "0"', 'CARGO_PROFILE_RELEASE_LTO: "off"', 'CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "8"'])
+ assert.ok(win7.includes(setting));
assert.ok(win7.includes("fc920bf0ec8de6ee65d409111f7ec508035751ba"));
assert.ok(win7.includes('version: "v0.16.0"'));
assert.ok(win7.includes("sccache --show-stats"));
@@ -432,7 +464,10 @@ test("Win7 TLS artifacts come from main and retain a build fallback", () => {
const win7 = job("windows-win7-bundle");
assert.ok(win7.includes("actions: read"));
assert.doesNotMatch(win7, /uses: actions\/cache(?:\/restore|\/save)?@v4/);
- for (const [name, id] of [["AWS-LC", "aws-lc-artifact"], ["OpenSSL", "openssl-artifact"]]) {
+ for (const [name, id] of [
+ ["AWS-LC", "aws-lc-artifact"],
+ ["OpenSSL", "openssl-artifact"],
+ ]) {
assert.ok(win7.includes(`if: steps.${id}.outputs.run_id != ''`));
assert.ok(win7.includes(`if: steps.${id}.outputs.run_id == ''`));
assert.ok(win7.includes(`if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.${id}.outputs.run_id == ''`));
diff --git a/.github/scripts/prepare-webview2-win7-runtime.ps1 b/.github/scripts/prepare-webview2-win7-runtime.ps1
index 15e41f150e..d917080e38 100644
--- a/.github/scripts/prepare-webview2-win7-runtime.ps1
+++ b/.github/scripts/prepare-webview2-win7-runtime.ps1
@@ -7,6 +7,64 @@ param(
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
+function Get-WebView2Archive {
+ param([string]$Uri, [string]$OutFile)
+
+ for ($attempt = 0; $attempt -lt 3; $attempt++) {
+ try {
+ Invoke-WebRequest -Uri $Uri -OutFile $OutFile -TimeoutSec 120
+ return
+ } catch {
+ $errorObject = $_.Exception
+ $retryable = $false
+ if ($errorObject.PSObject.Properties["Response"] -and $null -ne $errorObject.Response) {
+ $retryable = [int]$errorObject.Response.StatusCode -in @(408, 429, 500, 502, 503, 504)
+ } elseif ($errorObject -is [System.Net.Http.HttpRequestException] -and $null -ne $errorObject.StatusCode) {
+ $retryable = [int]$errorObject.StatusCode -in @(408, 429, 500, 502, 503, 504)
+ } elseif ($errorObject -is [System.Net.Http.HttpRequestException]) {
+ $networkCause = $false
+ $unsafeCause = $false
+ for ($cause = $errorObject.InnerException; $null -ne $cause; $cause = $cause.InnerException) {
+ if ($cause -is [System.Security.Authentication.AuthenticationException] -or
+ $cause -is [System.OperationCanceledException]) {
+ $unsafeCause = $true
+ } elseif ($cause -is [System.Net.Sockets.SocketException]) {
+ $networkCause = $networkCause -or $cause.SocketErrorCode -in @(
+ [System.Net.Sockets.SocketError]::ConnectionRefused,
+ [System.Net.Sockets.SocketError]::ConnectionReset,
+ [System.Net.Sockets.SocketError]::ConnectionAborted,
+ [System.Net.Sockets.SocketError]::TimedOut,
+ [System.Net.Sockets.SocketError]::HostNotFound,
+ [System.Net.Sockets.SocketError]::TryAgain,
+ [System.Net.Sockets.SocketError]::NetworkUnreachable,
+ [System.Net.Sockets.SocketError]::HostUnreachable,
+ [System.Net.Sockets.SocketError]::NetworkDown
+ )
+ }
+ }
+ $retryable = $networkCause -and !$unsafeCause
+ } elseif ($errorObject -is [System.Net.WebException]) {
+ $retryable = $errorObject.Status -in @(
+ [System.Net.WebExceptionStatus]::Timeout,
+ [System.Net.WebExceptionStatus]::ConnectFailure,
+ [System.Net.WebExceptionStatus]::NameResolutionFailure,
+ [System.Net.WebExceptionStatus]::ConnectionClosed,
+ [System.Net.WebExceptionStatus]::ReceiveFailure,
+ [System.Net.WebExceptionStatus]::SendFailure
+ )
+ } elseif ($errorObject -is [System.Threading.Tasks.TaskCanceledException]) {
+ $retryable = $true
+ }
+ if (!$retryable -or $attempt -eq 2) { throw }
+ Write-Host "WebView2 archive acquisition failed temporarily; retry $($attempt + 2)/3."
+ if (Test-Path -LiteralPath $OutFile) {
+ Remove-Item -LiteralPath $OutFile -Force
+ }
+ Start-Sleep -Seconds ([math]::Pow(2, $attempt + 1))
+ }
+ }
+}
+
$runtimeVersion = "109.0.1518.78"
$runtimeFolderName = "Microsoft.WebView2.FixedVersionRuntime.$runtimeVersion.x64"
$archiveName = "$runtimeFolderName.cab"
@@ -32,7 +90,7 @@ if (Test-Path $archivePath) {
if (!(Test-Path $archivePath)) {
Write-Host "Downloading WebView2 fixed runtime $runtimeVersion for Windows 7..."
- Invoke-WebRequest -Uri $runtimeUrl -OutFile $archivePath
+ Get-WebView2Archive -Uri $runtimeUrl -OutFile $archivePath
}
$actualHash = (Get-FileHash -LiteralPath $archivePath -Algorithm SHA256).Hash.ToLowerInvariant()
diff --git a/.github/scripts/prepare-webview2-win7-runtime.test.ps1 b/.github/scripts/prepare-webview2-win7-runtime.test.ps1
new file mode 100644
index 0000000000..1a1fac36ef
--- /dev/null
+++ b/.github/scripts/prepare-webview2-win7-runtime.test.ps1
@@ -0,0 +1,91 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+$source = Join-Path $PSScriptRoot "prepare-webview2-win7-runtime.ps1"
+$tokens = $null
+$errors = $null
+$ast = [System.Management.Automation.Language.Parser]::ParseFile($source, [ref]$tokens, [ref]$errors)
+if ($errors.Count) { throw $errors[0] }
+$definition = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq "Get-WebView2Archive" }, $true)
+. ([scriptblock]::Create($definition.Extent.Text))
+
+$directory = Join-Path ([IO.Path]::GetTempPath()) "dbx-ci-webview-$([guid]::NewGuid())"
+New-Item -ItemType Directory -Path $directory | Out-Null
+$outFile = Join-Path $directory "archive.cab"
+
+function Invoke-WebRequest {
+ param($Uri, $OutFile, $TimeoutSec)
+ if ($TimeoutSec -ne 120) { throw "Download timeout changed" }
+ if ($global:dbxWebviewTestcalls -gt 0 -and (Test-Path -LiteralPath $OutFile)) { throw "Partial download was reused" }
+ $result = $global:dbxWebviewTestresponses[$global:dbxWebviewTestcalls]
+ $global:dbxWebviewTestcalls++
+ [IO.File]::WriteAllText($OutFile, "partial")
+ switch ($result) {
+ "ok" { [IO.File]::WriteAllText($OutFile, "complete"); return }
+ "timeout" { throw [Net.WebException]::new("timeout", [Net.WebExceptionStatus]::Timeout) }
+ "refused" { throw [Net.Http.HttpRequestException]::new("connection refused", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionRefused)) }
+ "dns" { throw [Net.Http.HttpRequestException]::new("name lookup failed", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::HostNotFound)) }
+ "socket-denied" { throw [Net.Http.HttpRequestException]::new("access denied", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::AccessDenied)) }
+ "cancel-socket" { throw [Net.Http.HttpRequestException]::new("cancelled", [OperationCanceledException]::new("cancelled", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionReset))) }
+ "tls-socket" { throw [Net.Http.HttpRequestException]::new("TLS failed", [Security.Authentication.AuthenticationException]::new("certificate failure", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionReset))) }
+ "tls" { throw [Net.WebException]::new("certificate failure", [Net.WebExceptionStatus]::TrustFailure) }
+ "cancel" { throw [OperationCanceledException]::new("cancelled") }
+ "unknown" { throw "Unknown failure" }
+ default { throw [Net.Http.HttpRequestException]::new("HTTP failure", $null, [Net.HttpStatusCode][int]$result) }
+ }
+}
+function Start-Sleep { param($Seconds) $global:dbxWebviewTestdelays += $Seconds }
+
+try {
+ foreach ($case in @(
+ @{ Responses = @("ok"); Count = 1; Delays = @(); Success = $true },
+ @{ Responses = @("503", "ok"); Count = 2; Delays = @(2); Success = $true },
+ @{ Responses = @("timeout", "429", "ok"); Count = 3; Delays = @(2, 4); Success = $true },
+ @{ Responses = @("refused", "ok"); Count = 2; Delays = @(2); Success = $true },
+ @{ Responses = @("dns", "ok"); Count = 2; Delays = @(2); Success = $true },
+ @{ Responses = @("socket-denied"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("cancel-socket"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("tls-socket"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("503", "503", "503"); Count = 3; Delays = @(2, 4); Success = $false },
+ @{ Responses = @("404"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("403"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("tls"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("cancel"); Count = 1; Delays = @(); Success = $false },
+ @{ Responses = @("unknown"); Count = 1; Delays = @(); Success = $false }
+ )) {
+ Remove-Item -LiteralPath $outFile -Force -ErrorAction SilentlyContinue
+ $global:dbxWebviewTestresponses = $case.Responses
+ $global:dbxWebviewTestcalls = 0
+ $global:dbxWebviewTestdelays = @()
+ $success = $true
+ try { Get-WebView2Archive -Uri "https://example.test/archive.cab" -OutFile $outFile } catch { $success = $false }
+ if ($success -ne $case.Success -or $global:dbxWebviewTestcalls -ne $case.Count -or
+ ($global:dbxWebviewTestdelays -join ",") -ne ($case.Delays -join ",")) {
+ throw "Unexpected acquisition result for $($case.Responses -join ','): success=$success calls=$global:dbxWebviewTestcalls delays=$global:dbxWebviewTestdelays"
+ }
+ }
+
+ # Exercise the actual entry point: downloaded corrupt bytes must not be retried or expanded.
+ Remove-Item -LiteralPath $outFile -Force
+ $global:dbxWebviewTestresponses = @("ok")
+ $global:dbxWebviewTestcalls = 0
+ $global:dbxWebviewTestdelays = @()
+ $failed = $false
+ try { & $source -RuntimeDirectory (Join-Path $directory "runtime") -DownloadDirectory $directory } catch {
+ $failed = $_.Exception.Message -like "*SHA-256 mismatch*"
+ }
+ if (!$failed -or $global:dbxWebviewTestcalls -ne 1) { throw "Corrupt archive did not fail once before extraction" }
+
+ function Get-FileHash { param($LiteralPath, $Algorithm) [pscustomobject]@{ Hash = "7622281cf83de1a35e3a471f432f7a897d65f0a7d3975df08512b7b253dd45c7" } }
+ function Get-AuthenticodeSignature { param($LiteralPath) [pscustomobject]@{ Status = [System.Management.Automation.SignatureStatus]::NotSigned; SignerCertificate = $null } }
+ $failed = $false
+ $global:dbxWebviewTestcalls = 0
+ try { & $source -RuntimeDirectory (Join-Path $directory "runtime") -DownloadDirectory $directory } catch {
+ $failed = $_.Exception.Message -like "*valid Microsoft signature*"
+ }
+ if (!$failed -or $global:dbxWebviewTestcalls -ne 0) { throw "Invalid signature did not fail before extraction without re-download" }
+ Write-Host "WebView2 acquisition and integrity tests passed."
+} finally {
+ $resolved = [IO.Path]::GetFullPath($directory)
+ if ((Split-Path -Parent $resolved) -ne ([IO.Path]::GetTempPath()).TrimEnd('\', '/')) { throw "Unexpected test cleanup path" }
+ Remove-Item -LiteralPath $resolved -Recurse -Force
+}
diff --git a/.github/workflows/cache-keepalive.yml b/.github/workflows/cache-keepalive.yml
index a3355228ee..a427479516 100644
--- a/.github/workflows/cache-keepalive.yml
+++ b/.github/workflows/cache-keepalive.yml
@@ -41,25 +41,13 @@ jobs:
- uses: actions/checkout@v7
- name: Install system dependencies
- run: |
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- update
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+ run: sudo python3 .github/scripts/ci-install-system-deps.py
# Pin mold because runner image updates can change bundled C++ links.
# See the ci.yml rust-test note for the pinned upstream build.
- name: Install mold linker
- run: |
- curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
- | sudo tar -C /usr/local --strip-components=1 -xz
- mold --version
+ timeout-minutes: 8
+ run: bash .github/scripts/ci-install-mold.sh
- name: Setup Rust
uses: dtolnay/[email protected]
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 745f863109..e9c694d8de 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -174,6 +174,19 @@ jobs:
- name: Docker mirror config tests
run: python3 .github/scripts/configure-docker-mirror.test.py
+ - name: APT acquisition tests
+ run: python3 .github/scripts/ci-install-system-deps.test.py
+
+ - name: Gradle acquisition tests
+ run: python3 .github/scripts/ci-gradle.test.py
+
+ - name: Mold acquisition tests
+ run: python3 .github/scripts/ci-install-mold.test.py
+
+ - name: WebView2 acquisition tests
+ shell: pwsh
+ run: ./.github/scripts/prepare-webview2-win7-runtime.test.ps1
+
- name: PowerShell MCP installer tests
shell: pwsh
run: ./.github/scripts/install-mcp-powershell.test.ps1
@@ -207,25 +220,13 @@ jobs:
cache: pnpm
- name: Install system dependencies
- run: |
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- update
- sudo timeout --kill-after=30s 20m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+ run: sudo python3 .github/scripts/ci-install-system-deps.py
# Pin mold because runner image updates can change bundled DuckDB C++ links.
# The pinned build prevents duckdb_create_config runtime failures.
- name: Install mold linker
- run: |
- curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
- | sudo tar -C /usr/local --strip-components=1 -xz
- mold --version
+ timeout-minutes: 8
+ run: bash .github/scripts/ci-install-mold.sh
- name: Install frontend dependencies
run: pnpm install --frozen-lockfile
@@ -906,17 +907,7 @@ jobs:
- uses: actions/checkout@v7
- name: Install system dependencies
- run: |
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- update
- sudo timeout --kill-after=30s 20m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+ run: sudo python3 .github/scripts/ci-install-system-deps.py
- name: Setup Rust
uses: dtolnay/[email protected]
@@ -1026,26 +1017,14 @@ jobs:
done
- name: Install system dependencies
- run: |
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- update
- sudo timeout --kill-after=30s 20m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+ run: sudo python3 .github/scripts/ci-install-system-deps.py
if: matrix.group != 'foundation'
# Pin mold because runner image updates can change bundled C++ links.
# See the packages job note for the pinned upstream build.
- name: Install mold linker
- run: |
- curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
- | sudo tar -C /usr/local --strip-components=1 -xz
- mold --version
+ timeout-minutes: 8
+ run: bash .github/scripts/ci-install-mold.sh
- name: Setup Rust
uses: dtolnay/[email protected]
@@ -1174,7 +1153,7 @@ jobs:
node .github/scripts/check-jdbc-plugin-version.mjs "$BASE_SHA" HEAD
- name: JDBC plugin package check
- run: ./plugins/jdbc/package.sh
+ run: python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh
- name: JDBC response recovery tests
run: bash ./plugins/jdbc/gradlew -p plugins/jdbc --no-daemon test --tests app.dbx.jdbc.LargeResponseRecoveryTest --tests app.dbx.jdbc.StatementCancelTest --tests app.dbx.jdbc.DeferredLobValuesTest
@@ -1203,7 +1182,7 @@ jobs:
env:
DBX_OFFLINE_JDBC_MAVEN_CACHE: ~/.cache/dbx-offline-jdbc-maven
run: |
- ./plugins/jdbc/package.sh
+ python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh
JDBC_VERSION="$(sed -nE "s/^version[[:space:]]*=[[:space:]]*'([^']+)'.*/\1/p" plugins/jdbc/build.gradle | head -n 1)"
node agents/scripts/build_offline_jdbc_payload.mjs \
release \
@@ -1330,8 +1309,10 @@ jobs:
- '.github/workflows/ci.yml'
jdbc:
- 'plugins/jdbc/**'
+ - '.github/scripts/ci-gradle*'
offline_jdbc:
- 'plugins/jdbc/**'
+ - '.github/scripts/ci-gradle*'
- 'agents/scripts/build_offline_jdbc_payload.mjs'
- 'agents/scripts/build_offline_zip.sh'
- 'agents/scripts/verify_offline_jdbc_release.mjs'
@@ -1464,9 +1445,7 @@ jobs:
with:
node-version: 22.23.3
- name: Install packaging tools
- run: |
- sudo apt-get update
- sudo apt-get install -y zstd
+ run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging
- name: Agent script tests
run: |
python3 -m unittest discover -s scripts -p '*_test.py'
@@ -1494,11 +1473,9 @@ jobs:
with:
cache-provider: basic
- name: Install packaging tools
- run: |
- sudo apt-get update
- sudo apt-get install -y zstd
+ run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging
- name: Java agent tests and packages
- run: ./gradlew test shadowJar --continue
+ run: python3 ../.github/scripts/ci-gradle.py agents -- ./gradlew test shadowJar --continue
- name: Agent jar validation
run: python3 scripts/validate_agent_jars.py
agent-rust:
@@ -1516,26 +1493,14 @@ jobs:
- uses: actions/checkout@v7
- name: Install system dependencies
if: matrix.driver == 'duckdb'
- run: |
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- update
- sudo timeout --kill-after=30s 10m apt-get \
- -o Acquire::Retries=3 \
- -o Acquire::http::Timeout=30 \
- -o Acquire::https::Timeout=30 \
- install -y libdbus-1-dev pkg-config
+ run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group duckdb
- name: Setup Rust
uses: dtolnay/[email protected]
# Pin mold because runner image updates can change bundled DuckDB C++ links.
# See the packages job note for the pinned upstream build.
- name: Install mold linker
- run: |
- curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
- | sudo tar -C /usr/local --strip-components=1 -xz
- mold --version
+ timeout-minutes: 8
+ run: bash ../.github/scripts/ci-install-mold.sh
- name: Setup sccache
uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
with: