Self-hosted workflows for coding-agent delivery. Orchestrate CodeBuddy, Codex and Claude Code with scripts and human approval gates. Includes Local workflows for coding assistants.
JavaScript agent-orchestrationagentic-workflowai-agentsai-codingclaude-codecode-reviewcodebuddycodexcoding-agentscursordeveloper-toolshuman-on-the-loopllmmcpmulti-agentsdlcself-hostedsopworkflow-automationworkflow-engine
fix(ci): make Gitleaks downloads resilient to HTTP 504 and rate limits (#123)
Robert0622ffaed1574eb4
diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.ymlindex 0b14252c..c6043a9c 100644--- a/.github/workflows/platform.yml+++ b/.github/workflows/platform.yml@@ -49,7 +49,11 @@ jobs:- run: go run golang.org/x/vuln/cmd/[email protected] ./...- run: python3 scripts/collect-go-notices.py- run: python3 scripts/check-public.py+ - name: Secret scanner download regression tests+ run: python3 scripts/test-scan-secrets.py -v- run: bash scripts/scan-secrets.sh+ env:+ LOOPFORGE_GITLEAKS_TOKEN: ${{ github.token }}browser:runs-on: ubuntu-latesttimeout-minutes: 30diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.ymlindex 95eb4c2d..df88b458 100644--- a/.github/workflows/validate.yml+++ b/.github/workflows/validate.yml@@ -60,3 +60,5 @@ jobs:with:fetch-depth: 0- run: bash scripts/scan-secrets.sh+ env:+ LOOPFORGE_GITLEAKS_TOKEN: ${{ github.token }}diff --git a/local/scripts/scan-secrets.sh b/local/scripts/scan-secrets.shindex 8f311f0e..2ed8e814 100755--- a/local/scripts/scan-secrets.sh+++ b/local/scripts/scan-secrets.sh@@ -3,6 +3,8 @@ set -euo pipefailROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"VERSION="8.30.1"+# Official GitHub release asset for gitleaks_8.30.1_linux_x64.tar.gz.+ASSET_ID="378332058"ARCHIVE_SHA256="551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb"if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then@@ -14,9 +16,21 @@ TMP_DIR="$(mktemp -d)"trap 'rm -rf "$TMP_DIR"' EXITARCHIVE="$TMP_DIR/gitleaks.tar.gz"-curl -fsSL \+# GitHub release downloads can return transient 5xx responses.+if ! curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \+ --connect-timeout 15 --max-time 60 \"https://github.com/gitleaks/gitleaks/releases/download/v${VERSION}/gitleaks_${VERSION}_linux_x64.tar.gz" \- -o "$ARCHIVE"+ -o "$ARCHIVE"; then+ # Authenticate the API request when available to avoid the anonymous rate limit.+ {+ printf '%s\n' 'Accept: application/octet-stream'+ if [[ -n "${LOOPFORGE_GITLEAKS_TOKEN:-}" ]]; then+ printf 'Authorization: Bearer %s\n' "$LOOPFORGE_GITLEAKS_TOKEN"+ fi+ } | curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \+ --connect-timeout 15 --max-time 60 --header @- \+ "https://api.github.com/repos/gitleaks/gitleaks/releases/assets/${ASSET_ID}" -o "$ARCHIVE"+fiprintf '%s %s\n' "$ARCHIVE_SHA256" "$ARCHIVE" | sha256sum -c -tar -xzf "$ARCHIVE" -C "$TMP_DIR" gitleaksdiff --git a/scripts/scan-secrets.sh b/scripts/scan-secrets.shindex 4cff109f..5242173f 100644--- a/scripts/scan-secrets.sh+++ b/scripts/scan-secrets.sh@@ -2,10 +2,25 @@set -euo pipefailcd "$(dirname "$0")/.."version=8.30.1+# Official GitHub release asset for gitleaks_8.30.1_linux_x64.tar.gz.+asset_id=378332058archive_sha256=551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470ebscan_dir="$(mktemp -d)"trap 'rm -rf "$scan_dir"' EXIT-curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${version}/gitleaks_${version}_linux_x64.tar.gz" -o "$scan_dir/gitleaks.tar.gz"+# GitHub release downloads can return transient 5xx responses.+if ! curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \+ --connect-timeout 15 --max-time 60 \+ "https://github.com/gitleaks/gitleaks/releases/download/v${version}/gitleaks_${version}_linux_x64.tar.gz" -o "$scan_dir/gitleaks.tar.gz"; then+ # Authenticate the API request when available to avoid the anonymous rate limit.+ {+ printf '%s\n' 'Accept: application/octet-stream'+ if [[ -n "${LOOPFORGE_GITLEAKS_TOKEN:-}" ]]; then+ printf 'Authorization: Bearer %s\n' "$LOOPFORGE_GITLEAKS_TOKEN"+ fi+ } | curl -fsSL --retry 4 --retry-delay 2 --retry-max-time 120 \+ --connect-timeout 15 --max-time 60 --header @- \+ "https://api.github.com/repos/gitleaks/gitleaks/releases/assets/${asset_id}" -o "$scan_dir/gitleaks.tar.gz"+fiprintf '%s %s\n' "$archive_sha256" "$scan_dir/gitleaks.tar.gz" | sha256sum -c -tar -xzf "$scan_dir/gitleaks.tar.gz" -C "$scan_dir" gitleaks# Build an export of the candidate tracked/untracked files, respecting ignores.diff --git a/scripts/test-scan-secrets.py b/scripts/test-scan-secrets.pynew file mode 100644index 00000000..ed65a830--- /dev/null+++ b/scripts/test-scan-secrets.py@@ -0,0 +1,108 @@+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path+import os+import shutil+import subprocess+import tempfile+import threading+import unittest+++ROOT = Path(__file__).resolve().parents[1]+SCRIPTS = ('scripts/scan-secrets.sh', 'local/scripts/scan-secrets.sh')+++class ScanDownloadTests(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.requests = []+ self.responses = {}+ case = self++ class Handler(BaseHTTPRequestHandler):+ def do_GET(self):+ case.requests.append((self.path, self.headers.get('Accept'),+ self.headers.get('Authorization')))+ status, body = case.responses[self.path].pop(0)+ self.send_response(status)+ self.send_header('Content-Length', str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, *args):+ pass++ self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)+ thread = threading.Thread(target=self.server.serve_forever, daemon=True)+ thread.start()+ self.addCleanup(thread.join)+ self.addCleanup(self.server.server_close)+ self.addCleanup(self.server.shutdown)+ wrapper = Path(self.temp.name) / 'curl'+ wrapper.write_text('''#!/usr/bin/env python3+import os, sys+args = sys.argv[1:]+for i, arg in enumerate(args):+ if arg.startswith('https://github.com/gitleaks/gitleaks/releases/download/'):+ args[i] = os.environ['SCAN_TEST_BASE'] + '/release'+ elif arg.startswith('https://api.github.com/repos/gitleaks/gitleaks/releases/assets/'):+ args[i] = os.environ['SCAN_TEST_BASE'] + '/api'+os.execv(os.environ['SCAN_TEST_CURL'], ['curl', *args])+''')+ wrapper.chmod(0o755)+ self.env = dict(os.environ, PATH=self.temp.name + os.pathsep + os.environ['PATH'],+ SCAN_TEST_BASE=f'http://127.0.0.1:{self.server.server_port}',+ SCAN_TEST_CURL=shutil.which('curl'))+ self.env.pop('LOOPFORGE_GITLEAKS_TOKEN', None)++ def scan(self, script, token=None):+ self.requests.clear()+ env = self.env.copy()+ if token:+ env['LOOPFORGE_GITLEAKS_TOKEN'] = token+ return subprocess.run(['bash', str(ROOT / script)], env=env,+ capture_output=True, text=True, timeout=20)++ def test_retries_transient_error_and_checks_primary_download_checksum(self):+ for script in SCRIPTS:+ with self.subTest(script=script):+ self.responses = {'/release': [(504, b'gateway timeout'), (200, b'invalid archive')]}+ result = self.scan(script)+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/release'])+ self.assertNotEqual(result.returncode, 0)+ self.assertIn('FAILED', result.stdout)++ def test_uses_api_after_release_failure_and_checks_fallback_checksum(self):+ for script in SCRIPTS:+ with self.subTest(script=script):+ self.responses = {'/release': [(404, b'missing')], '/api': [(200, b'invalid archive')]}+ result = self.scan(script)+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/api'])+ self.assertEqual(self.requests[-1][1], 'application/octet-stream')+ self.assertNotEqual(result.returncode, 0)+ self.assertIn('FAILED', result.stdout)++ def test_sends_the_token_only_to_the_api_fallback(self):+ for script in SCRIPTS:+ with self.subTest(script=script):+ self.responses = {'/release': [(404, b'missing')], '/api': [(200, b'invalid archive')]}+ result = self.scan(script, token='test-api-token')+ self.assertEqual(self.requests, [+ ('/release', '*/*', None),+ ('/api', 'application/octet-stream', 'Bearer test-api-token'),+ ])+ self.assertNotEqual(result.returncode, 0)+ self.assertIn('FAILED', result.stdout)++ def test_fails_when_both_download_paths_fail(self):+ for script in SCRIPTS:+ with self.subTest(script=script):+ self.responses = {'/release': [(404, b'missing')], '/api': [(404, b'missing')]}+ result = self.scan(script)+ self.assertEqual([path for path, _, _ in self.requests], ['/release', '/api'])+ self.assertEqual(result.returncode, 22)+++if __name__ == '__main__':+ unittest.main()