Skip to content

Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.

fix(ci): make Gitleaks downloads resilient to HTTP 504 and rate limits (#123)

Robert0622ffaed1574eb4
diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml
index 0b14252c..c6043a9c 100644
--- a/.github/workflows/platform.yml
+++ b/.github/workflows/platform.yml
@@ -49,7 +49,11 @@ jobs:
- run: go run golang.org/x/vuln/cmd/[email protected] ./...
- run: python3 scripts/collect-go-notices.py
- run: python3 scripts/check-public.py
+ - name: Secret scanner download regression tests
+ run: python3 scripts/test-scan-secrets.py -v
- run: bash scripts/scan-secrets.sh
+ env:
+ LOOPFORGE_GITLEAKS_TOKEN: ${{ github.token }}
browser:
runs-on: ubuntu-latest
timeout-minutes: 30
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
index 95eb4c2d..df88b458 100644
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@@ -60,3 +60,5 @@ jobs:
with:
fetch-depth: 0
- run: bash scripts/scan-secrets.sh
+ env:
+ LOOPFORGE_GITLEAKS_TOKEN: ${{ github.token }}
diff --git a/local/scripts/scan-secrets.sh b/local/scripts/scan-secrets.sh
index 8f311f0e..2ed8e814 100755
--- a/local/scripts/scan-secrets.sh
+++ b/local/scripts/scan-secrets.sh
@@ -3,6 +3,8 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
VERSION="8.30.1"
+# Official GitHub release asset for gitleaks_8.30.1_linux_x64.tar.gz.
+ASSET_ID="378332058"
ARCHIVE_SHA256="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb"
if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then
@@ -14,9 +16,21 @@ TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
ARCHIVE="$TMP_DIR/gitleaks.tar.gz"
-curl -fsSL \
+# GitHub release downloads can return transient 5xx responses.
+if ! curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \
+ --connect-timeout 15 --max-time 60 \
"https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/gitleaks_${VERSION}_linux_x64.tar.gz" \
- -o "$ARCHIVE"
+ -o "$ARCHIVE"; then
+ # Authenticate the API request when available to avoid the anonymous rate limit.
+ {
+ printf '%s\n' 'Accept: application/octet-stream'
+ if [[ -n "${LOOPFORGE_GITLEAKS_TOKEN:-}" ]]; then
+ printf 'Authorization: Bearer %s\n' "$LOOPFORGE_GITLEAKS_TOKEN"
+ fi
+ } | curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \
+ --connect-timeout 15 --max-time 60 --header @- \
+ "https://api.github.com/repos/gitleaks/gitleaks/releases/assets/${ASSET_ID}" -o "$ARCHIVE"
+fi
printf '%s %s\n' "$ARCHIVE_SHA256" "$ARCHIVE" | sha256sum -c -
tar -xzf "$ARCHIVE" -C "$TMP_DIR" gitleaks
diff --git a/scripts/scan-secrets.sh b/scripts/scan-secrets.sh
index 4cff109f..5242173f 100644
--- a/scripts/scan-secrets.sh
+++ b/scripts/scan-secrets.sh
@@ -2,10 +2,25 @@
set -euo pipefail
cd "$(dirname "$0")/.."
version=8.30.1
+# Official GitHub release asset for gitleaks_8.30.1_linux_x64.tar.gz.
+asset_id=378332058
archive_sha256=551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
scan_dir="$(mktemp -d)"
trap 'rm -rf "$scan_dir"' EXIT
-curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${version}/gitleaks_${version}_linux_x64.tar.gz" -o "$scan_dir/gitleaks.tar.gz"
+# GitHub release downloads can return transient 5xx responses.
+if ! curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \
+ --connect-timeout 15 --max-time 60 \
+ "https://github.com/gitleaks/gitleaks/releases/download/v${version}/gitleaks_${version}_linux_x64.tar.gz" -o "$scan_dir/gitleaks.tar.gz"; then
+ # Authenticate the API request when available to avoid the anonymous rate limit.
+ {
+ printf '%s\n' 'Accept: application/octet-stream'
+ if [[ -n "${LOOPFORGE_GITLEAKS_TOKEN:-}" ]]; then
+ printf 'Authorization: Bearer %s\n' "$LOOPFORGE_GITLEAKS_TOKEN"
+ fi
+ } | curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \
+ --connect-timeout 15 --max-time 60 --header @- \
+ "https://api.github.com/repos/gitleaks/gitleaks/releases/assets/${asset_id}" -o "$scan_dir/gitleaks.tar.gz"
+fi
printf '%s %s\n' "$archive_sha256" "$scan_dir/gitleaks.tar.gz" | sha256sum -c -
tar -xzf "$scan_dir/gitleaks.tar.gz" -C "$scan_dir" gitleaks
# Build an export of the candidate tracked/untracked files, respecting ignores.
diff --git a/scripts/test-scan-secrets.py b/scripts/test-scan-secrets.py
new file mode 100644
index 00000000..ed65a830
--- /dev/null
+++ b/scripts/test-scan-secrets.py
@@ -0,0 +1,108 @@
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+import os
+import shutil
+import subprocess
+import tempfile
+import threading
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPTS = ('scripts/scan-secrets.sh', 'local/scripts/scan-secrets.sh')
+
+
+class ScanDownloadTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.requests = []
+ self.responses = {}
+ case = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ case.requests.append((self.path, self.headers.get('Accept'),
+ self.headers.get('Authorization')))
+ status, body = case.responses[self.path].pop(0)
+ self.send_response(status)
+ self.send_header('Content-Length', str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *args):
+ pass
+
+ self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
+ thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ thread.start()
+ self.addCleanup(thread.join)
+ self.addCleanup(self.server.server_close)
+ self.addCleanup(self.server.shutdown)
+ wrapper = Path(self.temp.name) / 'curl'
+ wrapper.write_text('''#!/usr/bin/env python3
+import os, sys
+args = sys.argv[1:]
+for i, arg in enumerate(args):
+ if arg.startswith('https://github.com/gitleaks/gitleaks/releases/download/'):
+ args[i] = os.environ['SCAN_TEST_BASE'] + '/release'
+ elif arg.startswith('https://api.github.com/repos/gitleaks/gitleaks/releases/assets/'):
+ args[i] = os.environ['SCAN_TEST_BASE'] + '/api'
+os.execv(os.environ['SCAN_TEST_CURL'], ['curl', *args])
+''')
+ wrapper.chmod(0o755)
+ self.env = dict(os.environ, PATH=self.temp.name + os.pathsep + os.environ['PATH'],
+ SCAN_TEST_BASE=f'http://127.0.0.1:{self.server.server_port}',
+ SCAN_TEST_CURL=shutil.which('curl'))
+ self.env.pop('LOOPFORGE_GITLEAKS_TOKEN', None)
+
+ def scan(self, script, token=None):
+ self.requests.clear()
+ env = self.env.copy()
+ if token:
+ env['LOOPFORGE_GITLEAKS_TOKEN'] = token
+ return subprocess.run(['bash', str(ROOT / script)], env=env,
+ capture_output=True, text=True, timeout=20)
+
+ def test_retries_transient_error_and_checks_primary_download_checksum(self):
+ for script in SCRIPTS:
+ with self.subTest(script=script):
+ self.responses = {'/release': [(504, b'gateway timeout'), (200, b'invalid archive')]}
+ result = self.scan(script)
+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/release'])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('FAILED', result.stdout)
+
+ def test_uses_api_after_release_failure_and_checks_fallback_checksum(self):
+ for script in SCRIPTS:
+ with self.subTest(script=script):
+ self.responses = {'/release': [(404, b'missing')], '/api': [(200, b'invalid archive')]}
+ result = self.scan(script)
+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/api'])
+ self.assertEqual(self.requests[-1][1], 'application/octet-stream')
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('FAILED', result.stdout)
+
+ def test_sends_the_token_only_to_the_api_fallback(self):
+ for script in SCRIPTS:
+ with self.subTest(script=script):
+ self.responses = {'/release': [(404, b'missing')], '/api': [(200, b'invalid archive')]}
+ result = self.scan(script, token='test-api-token')
+ self.assertEqual(self.requests, [
+ ('/release', '*/*', None),
+ ('/api', 'application/octet-stream', 'Bearer test-api-token'),
+ ])
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('FAILED', result.stdout)
+
+ def test_fails_when_both_download_paths_fail(self):
+ for script in SCRIPTS:
+ with self.subTest(script=script):
+ self.responses = {'/release': [(404, b'missing')], '/api': [(404, b'missing')]}
+ result = self.scan(script)
+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/api'])
+ self.assertEqual(result.returncode, 22)
+
+
+if __name__ == '__main__':
+ unittest.main()