Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(platform): pull full-stack CI's service images with a retry that outlasts a Docker Hub refusal (#15978)

Reinier van der Leerf47204a61e49
diff --git a/.github/scripts/docker-pull-with-retry.sh b/.github/scripts/docker-pull-with-retry.sh
index 791c0a30a7..d5578655a6 100644
--- a/.github/scripts/docker-pull-with-retry.sh
+++ b/.github/scripts/docker-pull-with-retry.sh
@@ -8,7 +8,8 @@ if [[ "$#" -ne 1 || -z "$1" ]]; then
fi
readonly image="$1"
-readonly max_attempts=3
+# Docker Hub has refused every pull for 20+ s at a time; five tries span 150 s.
+readonly max_attempts=5
for ((attempt = 1; attempt <= max_attempts; attempt++)); do
echo "Pulling ${image} (attempt ${attempt}/${max_attempts})"
@@ -21,7 +22,7 @@ for ((attempt = 1; attempt <= max_attempts; attempt++)); do
exit 1
fi
- delay_seconds=$((attempt * 2))
+ delay_seconds=$((attempt * 15))
echo "Retrying ${image} in ${delay_seconds}s" >&2
sleep "$delay_seconds"
done
diff --git a/.github/scripts/test_docker_pull_retry.py b/.github/scripts/test_docker_pull_retry.py
index d183a58188..40c6faca53 100644
--- a/.github/scripts/test_docker_pull_retry.py
+++ b/.github/scripts/test_docker_pull_retry.py
@@ -75,20 +75,20 @@ class DockerPullRetryTests(unittest.TestCase):
)
return result, count, delays, argument
- def test_success_after_zero_one_or_two_failures(self):
- for failures in (0, 1, 2):
+ def test_success_after_zero_one_or_four_failures(self):
+ for failures in (0, 1, 4):
with self.subTest(failures=failures):
result, count, delays, argument = self.run_pull(failures, ["image:tag"])
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(count, failures + 1)
- self.assertEqual(delays, ["2", "4"][:failures])
+ self.assertEqual(delays, ["15", "30", "45", "60"][:failures])
self.assertEqual(argument, "image:tag")
def test_terminal_failure_is_not_reported_as_success(self):
- result, count, delays, _ = self.run_pull(3, ["image:tag"])
+ result, count, delays, _ = self.run_pull(5, ["image:tag"])
self.assertEqual(result.returncode, 1, result.stderr)
- self.assertEqual(count, 3)
- self.assertEqual(delays, ["2", "4"])
+ self.assertEqual(count, 5)
+ self.assertEqual(delays, ["15", "30", "45", "60"])
def test_invalid_arguments_never_call_docker(self):
for args in ([], [""], ["one", "two"]):
diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.yml
index 5426309813..e66377d14a 100644
--- a/.github/workflows/platform-fullstack-ci.yml
+++ b/.github/workflows/platform-fullstack-ci.yml
@@ -387,10 +387,16 @@ jobs:
NEXT_PUBLIC_PW_TEST: true
NEXT_PUBLIC_SOURCEMAPS: true
- - name: Set up Platform - Start database
+ - name: Set up Platform - Pull service images and start database
id: database
background: true
run: |
+ # Every image `up` would pull, pulled now through the retry, so a brief
+ # Docker Hub refusal can't fail a later `up`.
+ images=$(python -c "import yaml; s = yaml.safe_load(open('../docker-compose.resolved.yml'))['services']; built = {c.get('image') for c in s.values() if 'build' in c}; print(' '.join(sorted({c['image'] for c in s.values() if 'build' not in c} - built)))")
+ for image in $images; do
+ bash "$GITHUB_WORKSPACE/.github/scripts/docker-pull-with-retry.sh" "$image"
+ done
docker compose -f ../docker-compose.resolved.yml up -d db --no-build
echo "Waiting for database to be ready..."
timeout 60 sh -c 'until docker compose -f ../docker-compose.resolved.yml exec -T db pg_isready -U postgres 2>/dev/null; do sleep 2; done'