Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Recover bounded HTTP2 module proxy failures in Docker builds

pulse-triage[bot]697608448a38
diff --git a/Dockerfile b/Dockerfile
index 4385fce29d..f072701ddd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -44,9 +44,10 @@ RUN apk add --no-cache git openssh-client
# Copy go mod files for better layer caching
COPY go.mod go.sum ./
+COPY scripts/go-mod-download.sh /usr/local/bin/pulse-go-mod-download
RUN --mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod \
--mount=type=cache,id=pulse-go-build,target=/root/.cache/go-build \
- go mod download
+ sh /usr/local/bin/pulse-go-mod-download
# Copy only necessary source code
COPY cmd/ ./cmd/
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index fb72048be3..ad637f729b 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -7013,3 +7013,23 @@ not gain a new full native matrix trigger.
either omitted graph input or a later exclusion. Existing real-build ordering,
Linux helper, Windows exit/service and installer checks remain unchanged. This
is admission coverage, not a local Windows/macOS verdict or installed recovery.
+
+### Bounded Docker module-proxy recovery (7 October 2026)
+
+The Docker backend builder uses `scripts/go-mod-download.sh` with the same
+`go.mod`, `go.sum`, Go toolchain, proxy/sum database and module/build caches.
+Only a failed download whose every diagnostic matches the observed HTTPS
+HTTP/2 `stream error ... INTERNAL_ERROR; received from peer` may run again.
+There are at most three attempts, each bounded to 180 seconds, with two- and
+four-second delays. Every failed attempt's output remains visible. Access
+refusals, checksum/integrity errors, unknown or mixed diagnostics, missing tools,
+timeouts and signals stop with the observed nonzero exit; an exhausted transport
+failure never becomes a success. No alternate endpoint or graph rewrite exists.
+
+`test_go_mod_download.py` executes the actual Docker RUN command and POSIX shell
+helper with controlled Go outcomes, including recovery, exhaustion and refusal
+precedence. Script smoke discovers it automatically. The existing Docker build
+contract checks the helper staging, unchanged cache/graph inputs and bounded
+command. These are offline command proofs, not a hosted Docker build, public
+proxy recovery, E2E acceptance or installed customer result. The hosted checks
+still have to build the image and execute the unchanged suites.
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 99b5dfdf3b..4b0ba088ef 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4546,6 +4546,7 @@
"scripts/dev-launchd-setup.sh",
"scripts/dev-launchd-wrapper.sh",
"scripts/generate-release-notes.sh",
+ "scripts/go-mod-download.sh",
"scripts/hot-dev-bg.sh",
"scripts/hot-dev.sh",
"scripts/install-container-agent.sh",
@@ -5137,6 +5138,19 @@
".github/scripts/tests/test_demo_runtime_transaction.py",
"scripts/installtests/demo_transaction_test.go"
]
+ },
+ {
+ "id": "go-module-download-runtime",
+ "label": "exact Docker Go module-download recovery proof",
+ "match_prefixes": [],
+ "match_files": [
+ "scripts/go-mod-download.sh"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "scripts/tests/test_go_mod_download.py"
+ ]
}
],
"match_files": null
diff --git a/scripts/go-mod-download.sh b/scripts/go-mod-download.sh
new file mode 100644
index 0000000000..34249df613
--- /dev/null
+++ b/scripts/go-mod-download.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+# Recover only the observed HTTP/2 proxy stream failure, never an access,
+# checksum, manifest or unknown error. Keep the same Go proxy and sumdb.
+set -eu
+
+if [ "$#" -ne 0 ]; then
+ echo 'Usage: go-mod-download.sh (no arguments)' >&2
+ exit 2
+fi
+
+log=$(mktemp)
+trap 'rm "$log"' 0
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+attempt=1
+while :; do
+ # Both Alpine BusyBox and GNU timeout support this fixed invocation.
+ # An attempt timeout is a terminal failure, not a reason to retry.
+ if timeout 180 go mod download >"$log" 2>&1; then
+ cat "$log"
+ exit 0
+ else
+ status=$?
+ fi
+ cat "$log" >&2
+ if [ "$status" -ne 1 ] || [ "$attempt" -eq 3 ]; then
+ exit "$status"
+ fi
+ # Every diagnostic must be the specific transport failure. A refusal or
+ # integrity error alongside it takes precedence and stops this attempt.
+ if ! LC_ALL=C awk '
+ /^go: [^[:space:]]+@[^[:space:]]+: read "https:\/\/[^"[:space:]]+": stream error: stream ID [0-9]+; INTERNAL_ERROR; received from peer$/ { seen = 1; next }
+ { unknown = 1 }
+ END { exit !(seen && !unknown) }
+ ' "$log"; then
+ exit "$status"
+ fi
+ echo "Go module proxy stream failed; retrying the same download ($attempt/3)." >&2
+ sleep "$((attempt * 2))"
+ attempt=$((attempt + 1))
+done
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 19cc39fe89..e9e788797b 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -5359,3 +5359,20 @@ func TestCITestCompilationPreservesChecks(t *testing.T) {
}
}
}
+
+func TestDockerModuleDownloadKeepsPinnedGraphAndBoundedRecovery(t *testing.T) {
+ assertFileContainsAll(t, repoFile("Dockerfile"),
+ "COPY go.mod go.sum ./\nCOPY scripts/go-mod-download.sh /usr/local/bin/pulse-go-mod-download",
+ "--mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod",
+ "sh /usr/local/bin/pulse-go-mod-download",
+ )
+ assertFileContainsAll(t, repoFile("scripts", "go-mod-download.sh"),
+ "timeout 180 go mod download", "[ \"$attempt\" -eq 3 ]", "INTERNAL_ERROR; received from peer",
+ "exit !(seen && !unknown)",
+ )
+ // The automatic script-smoke discovery must execute the behavioural proof,
+ // not leave Docker's new recovery path guarded only by text assertions.
+ if _, err := os.Stat(repoFile("scripts", "tests", "test_go_mod_download.py")); err != nil {
+ t.Fatalf("missing actual-shell module-download proof: %v", err)
+ }
+}
diff --git a/scripts/tests/test_go_mod_download.py b/scripts/tests/test_go_mod_download.py
new file mode 100644
index 0000000000..4be41598ea
--- /dev/null
+++ b/scripts/tests/test_go_mod_download.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Execute the Docker module-download command with controlled Go outcomes."""
+
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[2]
+HELPER = ROOT / 'scripts/go-mod-download.sh'
+STREAM_ERROR = ('go: go.opentelemetry.io/[email protected]: read '
+ '"https://proxy.golang.org/go.opentelemetry.io/otel/@v/v1.44.0.zip": '
+ 'stream error: stream ID 693; INTERNAL_ERROR; received from peer\n')
+
+
+def docker_download_command():
+ # Execute the actual RUN command, omitting only Docker cache-mount syntax.
+ dockerfile = Path(os.environ.get('PULSE_TEST_DOCKERFILE', ROOT / 'Dockerfile')).read_text()
+ stage = dockerfile.split('COPY go.mod go.sum ./\n', 1)[1].split('\n\n', 1)[0]
+ command = stage.split('RUN ', 1)[1]
+ command = re.sub(r'--mount=[^\s]+\s*\\\n\s*', '', command).strip()
+ return command.replace('/usr/local/bin/pulse-go-mod-download', str(HELPER))
+
+
+class GoModDownloadTests(unittest.TestCase):
+ def run_download(self, outcomes, *, command=None, timeout_status=None, args=()):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ tools = root / 'tools'
+ tools.mkdir()
+ temporary = root / 'temporary'
+ temporary.mkdir()
+ (root / 'outcomes.json').write_text(json.dumps(outcomes))
+ go = tools / 'go'
+ go.write_text('''#!/usr/bin/env python3
+import json, os, pathlib, sys
+root = pathlib.Path(os.environ['DOWNLOAD_FIXTURE'])
+with (root / 'calls.jsonl').open('a') as f:
+ f.write(json.dumps(sys.argv[1:]) + '\\n')
+count = len((root / 'calls.jsonl').read_text().splitlines())
+outcomes = json.loads((root / 'outcomes.json').read_text())
+status, text = outcomes[min(count - 1, len(outcomes) - 1)]
+print(text, end='', file=sys.stderr)
+sys.exit(status)
+''')
+ go.chmod(0o755)
+ timeout = tools / 'timeout'
+ timeout.write_text('''#!/bin/sh
+printf '%s\\n' "$*" >> "$DOWNLOAD_FIXTURE/timeout.calls"
+[ "$1" = 180 ] || exit 99
+if [ -n "${FIXTURE_TIMEOUT_STATUS:-}" ]; then exit "$FIXTURE_TIMEOUT_STATUS"; fi
+shift
+exec "$@"
+''')
+ timeout.chmod(0o755)
+ sleep = tools / 'sleep'
+ sleep.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$DOWNLOAD_FIXTURE/sleep.calls"\n')
+ sleep.chmod(0o755)
+ env = {**os.environ, 'PATH': str(tools) + ':' + os.environ['PATH'],
+ 'DOWNLOAD_FIXTURE': str(root), 'TMPDIR': str(temporary)}
+ if timeout_status is not None:
+ env['FIXTURE_TIMEOUT_STATUS'] = str(timeout_status)
+ else:
+ env.pop('FIXTURE_TIMEOUT_STATUS', None)
+ argv = ['sh', '-c', command] if command else ['sh', str(HELPER), *args]
+ result = subprocess.run(argv, cwd=ROOT, env=env, capture_output=True,
+ text=True, timeout=10)
+ calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else []
+ delays = (root / 'sleep.calls').read_text().splitlines() if (root / 'sleep.calls').exists() else []
+ timeouts = (root / 'timeout.calls').read_text().splitlines() if (root / 'timeout.calls').exists() else []
+ self.assertEqual(list(temporary.iterdir()), [], 'temporary diagnostic retained')
+ self.assertTrue(all(call == ['mod', 'download'] for call in calls), calls)
+ return result, calls, delays, timeouts
+
+ def test_success_has_one_unchanged_download(self):
+ result, calls, delays, timeouts = self.run_download([(0, 'download complete\n')])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, 'download complete\n')
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(delays, [])
+ self.assertEqual(timeouts, ['180 go mod download'])
+
+ def test_docker_stage_recovers_observed_failure(self):
+ result, calls, delays, timeouts = self.run_download(
+ [(1, STREAM_ERROR), (0, '')], command=docker_download_command())
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(len(calls), 2)
+ self.assertEqual(delays, ['2'])
+ self.assertEqual(timeouts, ['180 go mod download'] * 2)
+ self.assertIn(STREAM_ERROR, result.stderr)
+
+ def test_two_failures_can_recover_on_last_attempt(self):
+ result, calls, delays, _ = self.run_download([(1, STREAM_ERROR), (1, STREAM_ERROR), (0, '')])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(len(calls), 3)
+ self.assertEqual(delays, ['2', '4'])
+ self.assertEqual(result.stderr.count(STREAM_ERROR), 2)
+
+ def test_repeated_failure_exhausts_fixed_budget_and_stays_failed(self):
+ result, calls, delays, _ = self.run_download([(1, STREAM_ERROR)])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(len(calls), 3)
+ self.assertEqual(delays, ['2', '4'])
+ self.assertEqual(result.stderr.count(STREAM_ERROR), 3)
+
+ def test_multiple_matching_module_failures_can_recover(self):
+ result, calls, _, _ = self.run_download([(1, STREAM_ERROR * 2), (0, '')])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(len(calls), 2)
+
+ def test_access_integrity_and_unknown_errors_never_retry(self):
+ for error in ['401 Unauthorized', '403 Forbidden', 'SECURITY ERROR',
+ 'checksum mismatch', 'unknown revision v9.0.0',
+ 'x509: certificate signed by unknown authority',
+ 'proxyconnect tcp: connection refused', '']:
+ for transcript in [error, STREAM_ERROR + error]:
+ with self.subTest(error=error, mixed=transcript.startswith(STREAM_ERROR)):
+ # An empty extra line is unknown too; it cannot disguise
+ # an access or integrity failure as a transport-only one.
+ result, calls, delays, _ = self.run_download([(1, transcript + '\n'), (0, '')])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(delays, [])
+
+ def test_empty_transcript_never_retries(self):
+ result, calls, delays, _ = self.run_download([(1, ''), (0, '')])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(delays, [])
+
+ def test_only_exact_https_stream_signature_retries(self):
+ for text in [STREAM_ERROR.replace('https:', 'http:'),
+ STREAM_ERROR.replace('INTERNAL_ERROR', 'REFUSED_STREAM'),
+ STREAM_ERROR.replace('received from peer', 'other failure'),
+ 'an unrelated log mentioning ' + STREAM_ERROR]:
+ with self.subTest(text=text):
+ result, calls, delays, _ = self.run_download([(1, text), (0, '')])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(delays, [])
+
+ def test_nonordinary_command_exit_is_preserved_without_retry(self):
+ for status in [2, 124, 126, 127, 130, 137, 143]:
+ with self.subTest(status=status):
+ result, calls, delays, _ = self.run_download([(status, STREAM_ERROR), (0, '')])
+ self.assertEqual(result.returncode, status)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(delays, [])
+
+ def test_timeout_or_missing_timeout_never_runs_a_second_download(self):
+ for status in [124, 127, 137, 143]:
+ with self.subTest(status=status):
+ result, calls, delays, timeouts = self.run_download([(0, '')], timeout_status=status)
+ self.assertEqual(result.returncode, status)
+ self.assertEqual(calls, [])
+ self.assertEqual(delays, [])
+ self.assertEqual(timeouts, ['180 go mod download'])
+
+ def test_arbitrary_arguments_are_rejected_before_download(self):
+ result, calls, delays, timeouts = self.run_download([(0, '')], args=('-x',))
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(calls, [])
+ self.assertEqual(delays, [])
+ self.assertEqual(timeouts, [])
+
+ def test_docker_copies_helper_before_download_and_has_no_proxy_override(self):
+ dockerfile = (ROOT / 'Dockerfile').read_text()
+ copy = 'COPY scripts/go-mod-download.sh /usr/local/bin/pulse-go-mod-download'
+ self.assertIn(copy, dockerfile)
+ self.assertLess(dockerfile.index(copy), dockerfile.index('sh /usr/local/bin/pulse-go-mod-download'))
+ source = HELPER.read_text()
+ self.assertNotIn('GOPROXY=', source)
+ self.assertNotIn('GOSUMDB=', source)
+ self.assertNotIn('go mod tidy', source)
+ self.assertNotIn('go get', source)
+ self.assertNotIn('chmod', source)
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)