Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

apt repository: survive cancelled jobs, partial publishes, rotation and pagination

Claude46c3d9859af6
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5b46bc26..6523ae0e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -429,13 +429,17 @@ jobs:
# long, so the repository lives in an S3-compatible bucket (R2, S3, or GCS through its interop
# API) served over HTTPS at APT_REPO_URL. The pool keeps every version; only the index is read
# back. packaging/linux/apt/build-apt-repo.sh checks the published index against the committed
- # public key before re-signing it, so a bucket edit cannot be laundered into a valid signature,
- # and refuses to replace a version that is already published.
+ # public key before re-signing it, so an edited index is never re-signed, and refuses to replace
+ # a version that is already published. An older index this key once signed does pass that check;
+ # the backfill below re-adds any recent release such a rollback dropped.
#
# The packages are the bytes goreleaser published, checked against checksums.txt, not a rebuild.
#
# Skipped with a notice until every APT_* secret and variable exists, and for prerelease tags,
- # which do not belong in `stable`. Serialized so two tags cannot race on the index.
+ # which do not belong in `stable`. Serialized so two tags cannot race on the index. GitHub
+ # cancels a pending job when a third joins the group, so serializing alone can drop a release;
+ # each run therefore also adds recent releases the index is missing (see "Fetch the released
+ # .deb files"), and a release whose run was cancelled is published by the next one.
apt-repository:
needs: goreleaser
runs-on: ubuntu-latest
@@ -459,6 +463,8 @@ jobs:
APT_S3_ENDPOINT: ${{ vars.APT_S3_ENDPOINT }}
APT_REPO_URL: ${{ vars.APT_REPO_URL }}
KEYRING_ASC: packaging/linux/apt/beacon-archive-keyring.asc
+ # How many recent stable releases each run checks for and adds when the index lacks them.
+ APT_BACKFILL_RELEASES: 10
steps:
- name: Decide whether to publish
id: gate
@@ -503,9 +509,13 @@ jobs:
sudo apt-get install -y -qq apt-utils dpkg-dev gnupg
aws --version
- # The keyring users install is the committed one, never one exported from the secret, and
- # build-apt-repo.sh fails unless the secret's key signs for it. Rotating the key is therefore a
- # reviewed commit, not a secret edit.
+ # The keyring users install is built from the committed public key file, never exported from
+ # the secret, and build-apt-repo.sh fails unless the secret's key signs for it. Rotating the
+ # key is therefore a reviewed commit, not a secret edit: add the new public key to
+ # beacon-archive-keyring.asc beside the old one, replace the secret, and remove the old key
+ # once a release signed by the new one is published. The published index is checked against
+ # every key in the file, so the first release after a rotation still accepts the index the
+ # old key signed.
- name: Import the signing key
if: steps.gate.outputs.publish == 'true'
id: key
@@ -516,59 +526,122 @@ jobs:
mkdir -m 0700 "$GNUPGHOME"
echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV"
printf '%s\n' "$APT_GPG_PRIVATE_KEY" | gpg --batch --quiet --import
- gpg --batch --yes --dearmor --output "$RUNNER_TEMP/beacon-archive-keyring.gpg" "$KEYRING_ASC"
- fpr="$(gpg --batch --with-colons --show-keys "$KEYRING_ASC" | awk -F: '$1 == "fpr" { print $10; exit }')"
- [ -n "$fpr" ] || { echo "no fingerprint in $KEYRING_ASC" >&2; exit 1; }
- echo "fingerprint=$fpr" >> "$GITHUB_OUTPUT"
- echo "signing with $fpr"
-
- - name: Fetch the released .deb files
- if: steps.gate.outputs.publish == 'true'
- env:
- VERSION: ${{ steps.gate.outputs.version }}
- run: |
- set -euo pipefail
- mkdir -p dist/apt-debs && cd dist/apt-debs
- gh release download "$TAG" --repo "$REPO" --pattern checksums.txt
- for arch in amd64 arm64; do
- gh release download "$TAG" --repo "$REPO" --pattern "beacon_${VERSION}_linux_${arch}.deb"
- done
- # Every .deb must be listed; --ignore-missing alone would pass an unlisted one.
- for f in beacon_*.deb; do
- awk -v name="$f" '$2 == name { found = 1 } END { exit !found }' checksums.txt \
- || { echo "$f is not in checksums.txt" >&2; exit 1; }
- done
- sha256sum --check --ignore-missing --strict checksums.txt
-
- # Only the signed index is read back. An empty prefix is the first publish; a failed listing
- # fails the job rather than starting the repository over.
+ # Sign with the key the secret holds, not the first one in the committed file, which
+ # during a rotation is the retired key.
+ sec="$(gpg --batch --with-colons --list-secret-keys | awk -F: '$1 == "sec" { n++; want = 1; next } want && $1 == "fpr" { print $10; want = 0 } END { if (n != 1) exit 1 }')" \
+ || { echo "APT_GPG_PRIVATE_KEY must hold exactly one secret key" >&2; exit 1; }
+ committed="$(gpg --batch --with-colons --show-keys "$KEYRING_ASC" | awk -F: '$1 == "pub" { want = 1; next } want && $1 == "fpr" { print $10; want = 0 }')"
+ [ -n "$committed" ] || { echo "no key in $KEYRING_ASC" >&2; exit 1; }
+ printf '%s\n' "$committed" | grep -qxF "$sec" \
+ || { echo "the key in APT_GPG_PRIVATE_KEY ($sec) is not in $KEYRING_ASC; commit its public key first" >&2; exit 1; }
+ gpg --batch --quiet --import "$KEYRING_ASC"
+ # shellcheck disable=SC2086 # one fingerprint per word
+ gpg --batch --export $committed > "$RUNNER_TEMP/beacon-archive-keyring.gpg"
+ echo "fingerprint=$sec" >> "$GITHUB_OUTPUT"
+ echo "signing with $sec; trusted keys: $(printf '%s\n' "$committed" | paste -sd' ')"
+
+ # Only the signed index is read back. A missing InRelease is the first publish (or a first
+ # publish that stopped before writing it, which this resumes); any other error fails the job
+ # rather than starting the repository over. Checking InRelease itself rather than listing
+ # the prefix keeps the answer independent of pagination.
+ #
+ # Each Packages file is read through the by-hash path that InRelease names, never the plain
+ # path: by-hash copies are written before InRelease and never replaced, so a publish that
+ # stopped partway (plain Packages already new, InRelease still old) leaves them consistent.
+ # build-apt-repo.sh checks the signature and these hashes.
- name: Fetch the published index
if: steps.gate.outputs.publish == 'true'
id: existing
run: |
set -euo pipefail
- # Counts Contents rather than reading KeyCount, which some S3-compatible stores omit; a
- # missing KeyCount read as zero would start the index over and drop every older version.
- # shellcheck disable=SC2016 # the backticks are a JMESPath literal, not a command
- count="$(aws s3api list-objects-v2 --bucket "$APT_S3_BUCKET" --prefix dists/stable/ --query 'length(Contents || `[]`)' --output text)"
- case "$count" in
- ''|*[!0-9]*) echo "unexpected object count '$count' for dists/stable/" >&2; exit 1 ;;
- esac
- if [ "$count" = "0" ]; then
- echo "::notice::the bucket has no published index; this is the first publish"
- echo "dir=" >> "$GITHUB_OUTPUT"
- exit 0
+ if ! aws s3api head-object --bucket "$APT_S3_BUCKET" --key dists/stable/InRelease >/dev/null 2>"$RUNNER_TEMP/head.err"; then
+ if grep -q '(404)' "$RUNNER_TEMP/head.err"; then
+ echo "::notice::the bucket has no published InRelease; this is the first publish"
+ echo "dir=" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ cat "$RUNNER_TEMP/head.err" >&2
+ echo "could not check dists/stable/InRelease" >&2
+ exit 1
fi
dir="$RUNNER_TEMP/apt-existing"
aws s3 cp --no-progress "s3://$APT_S3_BUCKET/dists/stable/InRelease" "$dir/InRelease"
for arch in amd64 arm64; do
rel="main/binary-$arch/Packages"
- # An architecture added after the first publish has no index yet; build-apt-repo.sh
- # fails if InRelease lists one that is missing here.
- aws s3 cp --no-progress "s3://$APT_S3_BUCKET/dists/stable/$rel" "$dir/$rel" || true
+ want="$(awk -v rel="$rel" '/^[^ ]/ { in_sha = ($0 == "SHA256:"); next } in_sha && $3 == rel { print $1 }' "$dir/InRelease")"
+ # An architecture added after the first publish has no index yet.
+ [ -n "$want" ] || continue
+ case "$want" in *[!0-9a-f]*) echo "malformed SHA256 for $rel in InRelease" >&2; exit 1 ;; esac
+ aws s3 cp --no-progress "s3://$APT_S3_BUCKET/dists/stable/main/binary-$arch/by-hash/SHA256/$want" "$dir/$rel"
done
echo "dir=$dir" >> "$GITHUB_OUTPUT"
+ # This tag's packages, plus those of any of the last APT_BACKFILL_RELEASES stable releases up
+ # to this one that the index lacks. A concurrency group keeps one job running and one
+ # pending, and cancels the pending one when a third tag arrives, so a release tagged close to
+ # two others can lose its run; the next run adds it. The same repairs a first publish that
+ # stopped partway and an index rolled back in the bucket, within the window.
+ - name: Fetch the released .deb files
+ if: steps.gate.outputs.publish == 'true'
+ env:
+ VERSION: ${{ steps.gate.outputs.version }}
+ EXISTING: ${{ steps.existing.outputs.dir }}
+ run: |
+ set -euo pipefail
+ published() {
+ [ -n "$EXISTING" ] && [ -f "$EXISTING/main/binary-$2/Packages" ] || return 1
+ awk -v v="$1" '
+ BEGIN { RS = "" }
+ {
+ p = ver = ""
+ n = split($0, line, "\n")
+ for (i = 1; i <= n; i++) {
+ if (line[i] ~ /^Package: /) p = substr(line[i], 10)
+ else if (line[i] ~ /^Version: /) ver = substr(line[i], 10)
+ }
+ if (p == "beacon" && ver == v) found = 1
+ }
+ END { exit !found }' "$EXISTING/main/binary-$2/Packages"
+ }
+ # fetch TAG ARCH... -- downloads TAG's .debs for ARCH into dist/apt-debs/TAG and checks
+ # them against that release's checksums.txt.
+ fetch() {
+ tag="$1" version="${1#v}" dir="dist/apt-debs/$1"
+ shift
+ mkdir -p "$dir"
+ for arch in "$@"; do
+ gh release download "$tag" --repo "$REPO" --dir "$dir" --pattern "beacon_${version}_linux_${arch}.deb"
+ done
+ gh release download "$tag" --repo "$REPO" --dir "$dir" --pattern checksums.txt
+ # Every .deb must be listed; --ignore-missing alone would pass an unlisted one.
+ for f in "$dir"/beacon_*.deb; do
+ awk -v name="${f##*/}" '$2 == name { found = 1 } END { exit !found }' "$dir/checksums.txt" \
+ || { echo "${f##*/} is not in $tag's checksums.txt" >&2; exit 1; }
+ done
+ (cd "$dir" && sha256sum --check --ignore-missing --strict checksums.txt)
+ }
+ # This tag's packages are always fetched, so a rerun checks them against the index again.
+ fetch "$TAG" amd64 arm64
+ tags="$(gh release list --repo "$REPO" --exclude-drafts --exclude-pre-releases --limit 200 \
+ --json tagName --jq '.[].tagName' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || true)"
+ # Releases up to this one only: a newer tag's assets may still be uploading.
+ window="$(printf '%s\n%s\n' "$tags" "$TAG" | grep -v '^$' | sort -uV \
+ | awk -v cur="$TAG" '{ print } $0 == cur { exit }' | tail -n "$APT_BACKFILL_RELEASES")"
+ for tag in $window; do
+ [ "$tag" = "$TAG" ] && continue
+ assets="$(gh release view "$tag" --repo "$REPO" --json assets --jq '.assets[].name')"
+ missing=""
+ for arch in amd64 arm64; do
+ # Only what the release has: one without a .deb for an architecture is not retried.
+ printf '%s\n' "$assets" | grep -qxF "beacon_${tag#v}_linux_${arch}.deb" || continue
+ published "${tag#v}" "$arch" || missing="$missing $arch"
+ done
+ [ -n "$missing" ] || continue
+ echo "::notice::$tag is missing from the apt repository for$missing; adding it"
+ # shellcheck disable=SC2086 # one architecture per word
+ fetch "$tag" $missing
+ done
+
- name: Build and sign the repository update
if: steps.gate.outputs.publish == 'true'
env:
@@ -580,7 +653,7 @@ jobs:
if [ -n "$EXISTING" ]; then
set -- "$@" --existing "$EXISTING"
fi
- sh packaging/linux/apt/build-apt-repo.sh "$@" dist/apt-debs/beacon_*.deb
+ sh packaging/linux/apt/build-apt-repo.sh "$@" dist/apt-debs/*/beacon_*.deb
cp "$KEYRING_ASC" dist/apt-repo/beacon-archive-keyring.asc
cp "$RUNNER_TEMP/beacon-archive-keyring.gpg" dist/apt-repo/beacon-archive-keyring.gpg
diff --git a/CLAUDE.md b/CLAUDE.md
index 8ffd0fd9..72017386 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -291,10 +291,17 @@ A pushed `v*` tag runs the full release end-to-end via
released `.deb` files, checked against `checksums.txt`, to the signed apt repository
(`deb <APT_REPO_URL> stable main`) in an S3-compatible bucket, because 85-90 MB packages
outgrow a git-backed Pages repo. `packaging/linux/apt/build-apt-repo.sh` reads back only the
- published index, verifies it against the committed public key
- `packaging/linux/apt/beacon-archive-keyring.asc` before re-signing (so a bucket edit is never
- laundered into a valid signature), refuses to replace a published version, and fails unless
- the secret's key signs for the committed one, so rotating the key is a reviewed commit.
+ published index (InRelease, and each `Packages` through the by-hash path it names, so a
+ publish that stopped partway reads consistently and resumes), verifies it against the
+ committed public keys `packaging/linux/apt/beacon-archive-keyring.asc` before re-signing (so
+ an edited index is never re-signed), refuses to replace a published version, and fails unless
+ the secret's key is one of the committed ones. Rotating the key is a reviewed commit: add the
+ new public key beside the old, replace the secret (the job signs with the key the secret
+ holds), and drop the old key after one release. First publish is decided by a `head-object` on
+ `dists/stable/InRelease` (404 only; any other error fails). Each run adds its own tag plus any
+ of the last `APT_BACKFILL_RELEASES` (10) stable releases up to it that the index lacks, because
+ the `apt-repository` concurrency group cancels a pending job when a third arrives; that also
+ repairs an index rolled back in the bucket within the window.
`verify-apt-repo.sh` runs a real, isolated `apt-get update`/`download` against the staged
tree and again against the public URL; CI's `apt-repository` job runs `test-apt-repo.sh`.
Upload order is pool, index, then `Release.gpg`/`Release`/`InRelease` with `Acquire-By-Hash`.
diff --git a/packaging/linux/apt/build-apt-repo.sh b/packaging/linux/apt/build-apt-repo.sh
index 800a9471..db61df7d 100755
--- a/packaging/linux/apt/build-apt-repo.sh
+++ b/packaging/linux/apt/build-apt-repo.sh
@@ -6,9 +6,11 @@
#
# The published repository lives in object storage and keeps every version's .deb under pool/, so
# this script never downloads the pool. It takes the published index instead (--existing: a copy
-# of dists/stable holding InRelease and main/binary-<arch>/Packages), checks that index against
-# the committed public key (--keyring, a binary keyring), adds the new packages to it, and signs
-# the result with --signing-key. Without --existing it starts an empty repository, which is right
+# of dists/stable holding InRelease and main/binary-<arch>/Packages, the latter fetched through the
+# by-hash path InRelease names so a publish that stopped partway still reads consistently), checks
+# that index against the committed public keys (--keyring, a binary keyring that may hold the old
+# and new key during a rotation), adds the new packages to it, and signs the result with
+# --signing-key, which must be one of those keys. Without --existing it starts an empty repository, which is right
# only for the very first publish.
#
# Checking the published index first matters because the result is re-signed: a Packages file
@@ -96,7 +98,7 @@ if [ -n "$existing" ]; then
[ -f "$existing/InRelease" ] || fail "$existing/InRelease does not exist"
gpgv --keyring "$keyring" --output "$work/old-Release" "$existing/InRelease" 2>"$work/gpgv.log" || {
cat "$work/gpgv.log" >&2
- fail "the published InRelease is not signed by the committed key"
+ fail "the published InRelease is not signed by any committed key"
}
for arch in $ARCHES; do
rel="$COMPONENT/binary-$arch/Packages"
diff --git a/packaging/linux/apt/test-apt-repo.sh b/packaging/linux/apt/test-apt-repo.sh
index 2fadb218..c6ccff8c 100755
--- a/packaging/linux/apt/test-apt-repo.sh
+++ b/packaging/linux/apt/test-apt-repo.sh
@@ -7,6 +7,7 @@
#
# It publishes two releases in sequence and checks that both stay installable, that a rerun is a
# no-op, and that a replaced package, a tampered index and the wrong signing key are all refused.
+# It also resumes a publish that stopped before InRelease and rotates the signing key.
set -eu
HERE="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"
@@ -58,14 +59,17 @@ publish() {
done
}
-# existing -- the copy of the published index the release workflow downloads.
+# existing -- the copy of the published index the release workflow downloads: InRelease, and
+# each Packages file through the by-hash path InRelease names, never the plain path.
existing() {
rm -rf "$work/existing"
mkdir -p "$work/existing/main"
cp "$work/bucket/dists/stable/InRelease" "$work/existing/"
for arch in amd64 arm64; do
+ rel="main/binary-$arch/Packages"
+ want="$(awk -v rel="$rel" '/^[^ ]/ { in_sha = ($0 == "SHA256:"); next } in_sha && $3 == rel { print $1 }' "$work/existing/InRelease")"
mkdir -p "$work/existing/main/binary-$arch"
- cp "$work/bucket/dists/stable/main/binary-$arch/Packages" "$work/existing/main/binary-$arch/"
+ cp "$work/bucket/dists/stable/main/binary-$arch/by-hash/SHA256/$want" "$work/existing/$rel"
done
}
@@ -137,7 +141,7 @@ if sh "$BUILD" --out "$work/out6" --keyring "$work/other-keyring.gpg" --signing-
--existing "$work/existing" "$deb_a3" "$deb_b3" 2>"$work/err6"; then
fail "an index signed by another key was accepted"
fi
-grep -q 'not signed by the committed key' "$work/err6" || { cat "$work/err6" >&2; fail "wrong error for another key's index"; }
+grep -q 'not signed by any committed key' "$work/err6" || { cat "$work/err6" >&2; fail "wrong error for another key's index"; }
pass "an index signed by another key is refused"
# 7. A signing key that is not the committed key is refused before anything is published.
@@ -154,4 +158,46 @@ if sh "$VERIFY" "file:$work/bucket" "$work/other-keyring.gpg" 1.0.1 >/dev/null 2
fi
pass "a client trusting another key refuses the repository"
+# 9. A publish that stopped after the index files and before InRelease is resumed: the plain
+# Packages is already ahead of InRelease, but the by-hash copies InRelease names are not.
+existing
+sh "$BUILD" --out "$work/out9" --keyring "$work/keyring.gpg" --signing-key "$fpr" \
+ --existing "$work/existing" "$deb_a3" "$deb_b3"
+cp -R "$work/out9/pool" "$work/bucket/"
+cp -R "$work/out9/dists/stable/main" "$work/bucket/dists/stable/"
+existing
+sh "$BUILD" --out "$work/out9b" --keyring "$work/keyring.gpg" --signing-key "$fpr" \
+ --existing "$work/existing" "$deb_a3" "$deb_b3"
+publish "$work/out9b"
+for v in 1.0.0 1.0.1 1.0.2; do
+ sh "$VERIFY" "file:$work/bucket" "$work/keyring.gpg" "$v" >/dev/null
+done
+pass "a publish that stopped before InRelease is resumed"
+
+# 10. Key rotation: the committed keyring holds the old and the new key and the new key signs.
+# The published index the old key signed is still accepted, and clients trusting both or only
+# the new key accept the result. Several releases merge in one run (a missed release added
+# by the next one).
+gpg --batch --export "$fpr" "$other_fpr" > "$work/both-keyring.gpg"
+existing
+deb_a4="$(make_deb 1.0.3 amd64)"
+deb_b4="$(make_deb 1.0.3 arm64)"
+deb_a5="$(make_deb 1.0.4 amd64)"
+deb_b5="$(make_deb 1.0.4 arm64)"
+sh "$BUILD" --out "$work/out10" --keyring "$work/both-keyring.gpg" --signing-key "$other_fpr" \
+ --existing "$work/existing" "$deb_a4" "$deb_b4" "$deb_a5" "$deb_b5"
+publish "$work/out10"
+for v in 1.0.0 1.0.3 1.0.4; do
+ sh "$VERIFY" "file:$work/bucket" "$work/both-keyring.gpg" "$v" >/dev/null
+done
+sh "$VERIFY" "file:$work/bucket" "$work/other-keyring.gpg" 1.0.4 >/dev/null
+pass "a rotated key re-signs the old key's index, keeping every version"
+
+existing
+deb_a6="$(make_deb 1.0.5 amd64)"
+deb_b6="$(make_deb 1.0.5 arm64)"
+sh "$BUILD" --out "$work/out11" --keyring "$work/other-keyring.gpg" --signing-key "$other_fpr" \
+ --existing "$work/existing" "$deb_a6" "$deb_b6"
+pass "after a rotation the old key can be removed from the committed keyring"
+
echo "all apt repository checks passed"