Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci: retry apt with backoff and fall back to archive.debian.org when snapshot.debian.org fails (#6582)

cua-agents[bot]9a9f975eb03e
diff --git a/.github/scripts/tests/test_driver_linux_bootstrap.py b/.github/scripts/tests/test_driver_linux_bootstrap.py
index fcd8d2aa54..6843eda7b8 100644
--- a/.github/scripts/tests/test_driver_linux_bootstrap.py
+++ b/.github/scripts/tests/test_driver_linux_bootstrap.py
@@ -53,6 +53,17 @@ class TestDriverLinuxBootstrap(unittest.TestCase):
suites.add(suite)
self.assertEqual(suites, {"bullseye", "bullseye-updates", "bullseye-security"})
remaining = self.bootstrap.replace(sources[1], "")
+ # The documented archive.debian.org fallback (past Valid-Until by
+ # design) is the only other place expiry checks may be disabled.
+ fallback = re.search(
+ r"cat > /etc/apt/sources.list <<'EOF'\n(deb \[check-valid-until=no\] "
+ r"http://archive.debian.org/.*?)\nEOF",
+ remaining,
+ re.S,
+ )
+ self.assertIsNotNone(fallback)
+ self.assertEqual(len(fallback[1].splitlines()), 3)
+ remaining = remaining.replace(fallback[1], "")
self.assertNotRegex(remaining.lower(), r"check-valid-until|check-date")
self.assertNotRegex(
self.bootstrap.lower(),
@@ -60,27 +71,28 @@ class TestDriverLinuxBootstrap(unittest.TestCase):
r"allowweak|allow-weak|verify-peer|verify-host",
)
- def test_apt_uses_only_snapshot_sources_and_full_dependencies(self) -> None:
- commands = [
- shlex.split(line)
- for line in self.bootstrap.replace("\\\n", " ").splitlines()
- if line.startswith("apt-get ")
- ]
- self.assertEqual(commands[0], ["apt-get", "-o", "Dir::Etc::sourceparts=-", "-o", "Acquire::Retries=5", "update"])
- self.assertEqual(
- commands[1][:8],
- ["apt-get", "-o", "Dir::Etc::sourceparts=-", "-o", "Acquire::Retries=5", "install", "-y", "--no-install-recommends"],
+ def test_apt_retries_snapshot_first_then_archive_fallback(self) -> None:
+ self.assertIn('Acquire::Retries "5";', self.bootstrap)
+ self.assertIn('Acquire::http::Timeout "30";', self.bootstrap)
+ self.assertIn("apt mirror used: snapshot.debian.org 20260901T000000Z", self.bootstrap)
+ self.assertIn("apt mirror used: archive.debian.org", self.bootstrap)
+ # Snapshot sources are written first, the archive fallback second.
+ self.assertLess(
+ self.bootstrap.index("snapshot.debian.org/archive/debian/"),
+ self.bootstrap.index("http://archive.debian.org/debian bullseye main"),
)
- self.assertEqual(len(commands), 2)
+ # Image-provided extra sources stay ignored in both attempts.
+ self.assertIn("Dir::Etc::sourceparts=-", self.bootstrap)
+ pkgs = re.search(r'PACKAGES="([^"]*)"', self.bootstrap)
+ self.assertIsNotNone(pkgs)
self.assertEqual(
- set(commands[1][8:]),
+ set(pkgs[1].split()),
{
"git", "ca-certificates", "curl", "python3", "build-essential", "pkg-config",
"libx11-dev", "libxi-dev", "libxtst-dev", "libxext-dev", "libwayland-dev",
"libxkbcommon-dev",
},
)
- self.assertLess(self.bootstrap.index("\nEOF\n"), self.bootstrap.index("apt-get "))
if __name__ == "__main__":
diff --git a/.github/workflows/cd-cua-sdk.yml b/.github/workflows/cd-cua-sdk.yml
index e6619b3007..2fae9e3317 100644
--- a/.github/workflows/cd-cua-sdk.yml
+++ b/.github/workflows/cd-cua-sdk.yml
@@ -159,23 +159,58 @@ jobs:
- name: Install base tooling (debian:11 is bare)
if: matrix.container != ''
run: |
- # The live bullseye-security mirror now 404s: freeze indexes and
- # packages on snapshot.debian.org (same pin as cd-rust-cua-driver.yml;
- # the glibc 2.31 floor is unchanged). HTTP bootstraps ca-certificates;
- # APT still verifies Debian signatures and hashes.
+ # Debian 11 is past its support window, so bullseye indexes and packages
+ # are frozen on snapshot.debian.org (the live security mirror advertises
+ # packages it no longer serves). HTTP bootstraps ca-certificates; APT still
+ # verifies Debian signatures and hashes.
+ # snapshot.debian.org answers 504/503 under load. APT retries each fetch
+ # with timeouts, the shell retries the whole step with a backoff, and if
+ # the snapshot is still down we fall back to archive.debian.org, which
+ # holds the same final bullseye packages (the pin exists for
+ # reproducibility, not for different versions). Same retry policy as
+ # libs/cua-spacesd/packaging/release/debian-build-deps.sh.
+ mkdir -p /etc/apt/apt.conf.d
+ cat > /etc/apt/apt.conf.d/80-cua-retries <<'EOF'
+ Acquire::Retries "5";
+ Acquire::http::Timeout "30";
+ Acquire::https::Timeout "30";
+ EOF
+ PACKAGES="git ca-certificates curl build-essential pkg-config unzip python3 xz-utils cmake"
+ apt_retry() {
+ n=0
+ until "$@"; do
+ n=$((n + 1))
+ if [ "$n" -ge 3 ]; then return 1; fi
+ echo "apt step failed (attempt $n of 3); retrying in $((n * 20))s" >&2
+ sleep $((n * 20))
+ done
+ }
+ apt_bootstrap() {
+ # Ignore any additional image-provided sources for this bootstrap.
+ apt_retry apt-get -o Dir::Etc::sourceparts=- "$@" update &&
+ apt_retry apt-get -o Dir::Etc::sourceparts=- "$@" install -y --no-install-recommends $PACKAGES
+ }
+ rm -f /etc/apt/sources.list.d/debian.sources
cat > /etc/apt/sources.list <<'EOF'
deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye main
deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye-updates main
deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian-security/20260901T000000Z/ bullseye-security main
EOF
- rm -f /etc/apt/sources.list.d/debian.sources
- # snapshot.debian.org answers 504 under load; retry each fetch.
- echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries
- apt-get update
- apt-get install -y --no-install-recommends git ca-certificates curl build-essential \
- pkg-config unzip python3 xz-utils cmake
+ if apt_bootstrap; then
+ echo "apt mirror used: snapshot.debian.org 20260901T000000Z"
+ else
+ echo "::warning::snapshot.debian.org failed after retries; falling back to archive.debian.org"
+ rm -rf /var/lib/apt/lists/*
+ cat > /etc/apt/sources.list <<'EOF'
+ deb [check-valid-until=no] http://archive.debian.org/debian bullseye main
+ deb [check-valid-until=no] http://archive.debian.org/debian bullseye-updates main
+ deb [check-valid-until=no] http://archive.debian.org/debian-security bullseye-security main
+ EOF
+ apt_bootstrap
+ echo "apt mirror used: archive.debian.org (snapshot fallback)"
+ fi
curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors --connect-timeout 20 https://deb.nodesource.com/setup_22.x | bash -
- apt-get install -y nodejs
+ apt_retry apt-get install -y nodejs
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Install protoc
uses: ./.github/actions/setup-protoc
diff --git a/.github/workflows/cd-rust-cua-driver.yml b/.github/workflows/cd-rust-cua-driver.yml
index 967b5c9b73..61e7ba0e8f 100644
--- a/.github/workflows/cd-rust-cua-driver.yml
+++ b/.github/workflows/cd-rust-cua-driver.yml
@@ -232,25 +232,60 @@ jobs:
# is bare. Install git + the build toolchain before anything else.
- name: Install base tooling (container is bare)
run: |
- # Freeze bullseye indexes and packages together: the live security
- # mirror can advertise packages it no longer serves. HTTP bootstraps
- # ca-certificates; APT still verifies Debian signatures and hashes.
- cat > /etc/apt/sources.list <<'EOF'
- deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye main
- deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye-updates main
- deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian-security/20260901T000000Z/ bullseye-security main
+ # Debian 11 is past its support window, so bullseye indexes and packages
+ # are frozen on snapshot.debian.org (the live security mirror advertises
+ # packages it no longer serves). HTTP bootstraps ca-certificates; APT still
+ # verifies Debian signatures and hashes.
+ # snapshot.debian.org answers 504/503 under load. APT retries each fetch
+ # with timeouts, the shell retries the whole step with a backoff, and if
+ # the snapshot is still down we fall back to archive.debian.org, which
+ # holds the same final bullseye packages (the pin exists for
+ # reproducibility, not for different versions). Same retry policy as
+ # libs/cua-spacesd/packaging/release/debian-build-deps.sh.
+ mkdir -p /etc/apt/apt.conf.d
+ cat > /etc/apt/apt.conf.d/80-cua-retries <<'EOF'
+ Acquire::Retries "5";
+ Acquire::http::Timeout "30";
+ Acquire::https::Timeout "30";
EOF
- # Ignore any additional image-provided sources for this bootstrap.
- # snapshot.debian.org answers 504 under load; retry each fetch.
- apt-get -o Dir::Etc::sourceparts=- -o Acquire::Retries=5 update
# PipeWire ScreenCast capture remains Nix/modern-build only because
# bullseye's 0.3.19 headers are too old for libspa-sys 0.10. Portal
# RemoteDesktop/libei input is pure Rust plus libxkbcommon and ships
# in these portable release binaries.
- apt-get -o Dir::Etc::sourceparts=- -o Acquire::Retries=5 install -y --no-install-recommends \
- git ca-certificates curl python3 build-essential pkg-config \
- libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \
- libxkbcommon-dev
+ PACKAGES="git ca-certificates curl python3 build-essential pkg-config libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev libxkbcommon-dev"
+ apt_retry() {
+ n=0
+ until "$@"; do
+ n=$((n + 1))
+ if [ "$n" -ge 3 ]; then return 1; fi
+ echo "apt step failed (attempt $n of 3); retrying in $((n * 20))s" >&2
+ sleep $((n * 20))
+ done
+ }
+ apt_bootstrap() {
+ # Ignore any additional image-provided sources for this bootstrap.
+ apt_retry apt-get -o Dir::Etc::sourceparts=- "$@" update &&
+ apt_retry apt-get -o Dir::Etc::sourceparts=- "$@" install -y --no-install-recommends $PACKAGES
+ }
+ rm -f /etc/apt/sources.list.d/debian.sources
+ cat > /etc/apt/sources.list <<'EOF'
+ deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye main
+ deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian/20260901T000000Z/ bullseye-updates main
+ deb [check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://snapshot.debian.org/archive/debian-security/20260901T000000Z/ bullseye-security main
+ EOF
+ if apt_bootstrap; then
+ echo "apt mirror used: snapshot.debian.org 20260901T000000Z"
+ else
+ echo "::warning::snapshot.debian.org failed after retries; falling back to archive.debian.org"
+ rm -rf /var/lib/apt/lists/*
+ cat > /etc/apt/sources.list <<'EOF'
+ deb [check-valid-until=no] http://archive.debian.org/debian bullseye main
+ deb [check-valid-until=no] http://archive.debian.org/debian bullseye-updates main
+ deb [check-valid-until=no] http://archive.debian.org/debian-security bullseye-security main
+ EOF
+ apt_bootstrap
+ echo "apt mirror used: archive.debian.org (snapshot fallback)"
+ fi
- uses: actions/checkout@v4
with:
ref: ${{ inputs.source_ref || github.ref }}
diff --git a/libs/cua-spacesd/packaging/release/debian-build-deps.sh b/libs/cua-spacesd/packaging/release/debian-build-deps.sh
index 377e558081..ebe7060b41 100755
--- a/libs/cua-spacesd/packaging/release/debian-build-deps.sh
+++ b/libs/cua-spacesd/packaging/release/debian-build-deps.sh
@@ -16,6 +16,15 @@
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
+# Shared retry policy: per-fetch retries and timeouts (also see the inline
+# bootstraps in cd-rust-cua-driver.yml and cd-cua-sdk.yml).
+mkdir -p /etc/apt/apt.conf.d
+cat > /etc/apt/apt.conf.d/80-cua-retries <<'EOF'
+Acquire::Retries "5";
+Acquire::http::Timeout "30";
+Acquire::https::Timeout "30";
+EOF
+
PACKAGES=(
git ca-certificates curl unzip xz-utils file binutils
build-essential pkg-config perl make cmake nasm clang
@@ -30,9 +39,10 @@ install_packages() {
}
if install_packages; then
+ echo "apt mirror used: image default (deb.debian.org)" >&2
exit 0
fi
-echo "falling back to archive.debian.org sources" >&2
+echo "::warning::default mirror failed after retries; falling back to archive.debian.org" >&2
cat > /etc/apt/sources.list <<'EOF'
deb [check-valid-until=no] http://archive.debian.org/debian bullseye main
deb [check-valid-until=no] http://archive.debian.org/debian bullseye-updates main
@@ -40,3 +50,4 @@ deb [check-valid-until=no] http://archive.debian.org/debian-security bullseye-se
EOF
rm -f /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources
install_packages -o Acquire::Check-Valid-Until=false -o Acquire::Retries=5
+echo "apt mirror used: archive.debian.org (fallback)" >&2