The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
feat(release): publish a signed apt repository on each release
Justin D'Souza76158aa1fd71
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 22eb0be3..d00441ab 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -754,6 +754,22 @@ jobs:BEACON_SMOKE_FORMAT: rpmrun: sh packaging/linux/smoke-package.sh+ # The scripts release.yml uses to publish the signed apt repository, run against stand-in+ # packages, a throwaway key and a local directory in place of the bucket. Checks a merge keeps+ # older versions installable and that a replaced package, a tampered index and the wrong key are+ # refused, all through a real apt-get running as an ordinary user.+ apt-repository:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@v7+ - name: Install apt repository tools+ run: |+ sudo apt-get update -qq+ sudo apt-get install -y -qq apt-utils dpkg-dev gnupg+ - name: Test the apt repository scripts+ run: sh packaging/linux/apt/test-apt-repo.sh+package-smoke:runs-on: macos-lateststeps:diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex e3f91ca2..5b46bc26 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -9,9 +9,12 @@ name: Release# 3. windows-package — (after goreleaser) builds the Windows MSI from the archive# goreleaser just published, installs it to prove it works, and# attaches it. Unsigned for now, so it needs no secrets.-# 4. finalize — (after all three) adds the .pkg, .msi and install.sh to checksums.txt and-# records a GitHub build-provenance attestation for every file-# checksums.txt lists.+# 4. apt-repository — (after goreleaser) adds the released .deb files to the signed apt+# repository in object storage. Skipped until its APT_* secrets and+# variables are configured.+# 5. finalize — (after goreleaser, package and windows-package) adds the .pkg, .msi and+# install.sh to checksums.txt and records a GitHub build-provenance+# attestation for every file checksums.txt lists.## Triggered only by pushed version tags, so outside contributors (whose fork PRs# never receive secrets) cannot run it. All release secrets live in the protected@@ -419,6 +422,216 @@ jobs:& gh release upload $env:TAG (Get-ChildItem dist\windows -Filter '*.msi').FullName (Get-ChildItem dist\windows -Filter '*.msi.sha256').FullName --repo $env:REPO --clobberif ($LASTEXITCODE -ne 0) { throw "uploading the MSI failed" }+ # The signed apt repository (`deb https://<APT_REPO_URL> stable main`), so Debian and Ubuntu hosts+ # can `apt install beacon` and get upgrades with `apt upgrade`.+ #+ # The .deb files are 85-90 MB each, past what a git-backed GitHub Pages repository can hold for+ # long, so the repository lives in an S3-compatible bucket (R2, S3, or GCS through its interop+ # API) served over HTTPS at APT_REPO_URL. The pool keeps every version; only the index is read+ # back. packaging/linux/apt/build-apt-repo.sh checks the published index against the committed+ # public key before re-signing it, so a bucket edit cannot be laundered into a valid signature,+ # and refuses to replace a version that is already published.+ #+ # The packages are the bytes goreleaser published, checked against checksums.txt, not a rebuild.+ #+ # Skipped with a notice until every APT_* secret and variable exists, and for prerelease tags,+ # which do not belong in `stable`. Serialized so two tags cannot race on the index.+ apt-repository:+ needs: goreleaser+ runs-on: ubuntu-latest+ environment: release+ timeout-minutes: 30+ concurrency:+ group: apt-repository+ cancel-in-progress: false+ env:+ REPO: ${{ github.repository }}+ TAG: ${{ github.ref_name }}+ GH_TOKEN: ${{ github.token }}+ APT_GPG_PRIVATE_KEY: ${{ secrets.APT_GPG_PRIVATE_KEY }}+ AWS_ACCESS_KEY_ID: ${{ secrets.APT_S3_ACCESS_KEY_ID }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.APT_S3_SECRET_ACCESS_KEY }}+ AWS_DEFAULT_REGION: ${{ vars.APT_S3_REGION || 'us-east-1' }}+ # R2 and GCS interop do not accept the CRC checksums newer AWS CLIs send by default.+ AWS_REQUEST_CHECKSUM_CALCULATION: when_required+ AWS_RESPONSE_CHECKSUM_VALIDATION: when_required+ APT_S3_BUCKET: ${{ vars.APT_S3_BUCKET }}+ APT_S3_ENDPOINT: ${{ vars.APT_S3_ENDPOINT }}+ APT_REPO_URL: ${{ vars.APT_REPO_URL }}+ KEYRING_ASC: packaging/linux/apt/beacon-archive-keyring.asc+ steps:+ - name: Decide whether to publish+ id: gate+ shell: bash+ run: |+ set -euo pipefail+ missing=""+ for name in APT_GPG_PRIVATE_KEY AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APT_S3_BUCKET APT_REPO_URL; do+ [ -n "${!name}" ] || missing="$missing $name"+ done+ if [ -n "$missing" ]; then+ echo "::notice title=apt repository skipped::not configured (missing:$missing). The release is unaffected."+ echo "publish=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ case "$TAG" in+ *-*)+ echo "::notice title=apt repository skipped::$TAG is a prerelease; only releases go to the stable suite."+ echo "publish=false" >> "$GITHUB_OUTPUT"+ exit 0+ ;;+ esac+ case "$APT_REPO_URL" in+ https://*) ;;+ *) echo "APT_REPO_URL must be https://" >&2; exit 1 ;;+ esac+ # Set only when present: the AWS CLI treats an empty AWS_ENDPOINT_URL as an endpoint.+ if [ -n "$APT_S3_ENDPOINT" ]; then+ echo "AWS_ENDPOINT_URL=$APT_S3_ENDPOINT" >> "$GITHUB_ENV"+ fi+ echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"+ echo "publish=true" >> "$GITHUB_OUTPUT"++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ if: steps.gate.outputs.publish == 'true'++ - name: Install apt repository tools+ if: steps.gate.outputs.publish == 'true'+ run: |+ set -euo pipefail+ sudo apt-get update -qq+ sudo apt-get install -y -qq apt-utils dpkg-dev gnupg+ aws --version++ # The keyring users install is the committed one, never one exported from the secret, and+ # build-apt-repo.sh fails unless the secret's key signs for it. Rotating the key is therefore a+ # reviewed commit, not a secret edit.+ - name: Import the signing key+ if: steps.gate.outputs.publish == 'true'+ id: key+ run: |+ set -euo pipefail+ [ -f "$KEYRING_ASC" ] || { echo "$KEYRING_ASC is not committed; commit the public key before configuring APT_GPG_PRIVATE_KEY" >&2; exit 1; }+ export GNUPGHOME="$RUNNER_TEMP/gnupg"+ mkdir -m 0700 "$GNUPGHOME"+ echo "GNUPGHOME=$GNUPGHOME" >> "$GITHUB_ENV"+ printf '%s\n' "$APT_GPG_PRIVATE_KEY" | gpg --batch --quiet --import+ gpg --batch --yes --dearmor --output "$RUNNER_TEMP/beacon-archive-keyring.gpg" "$KEYRING_ASC"+ fpr="$(gpg --batch --with-colons --show-keys "$KEYRING_ASC" | awk -F: '$1 == "fpr" { print $10; exit }')"+ [ -n "$fpr" ] || { echo "no fingerprint in $KEYRING_ASC" >&2; exit 1; }+ echo "fingerprint=$fpr" >> "$GITHUB_OUTPUT"+ echo "signing with $fpr"++ - name: Fetch the released .deb files+ if: steps.gate.outputs.publish == 'true'+ env:+ VERSION: ${{ steps.gate.outputs.version }}+ run: |+ set -euo pipefail+ mkdir -p dist/apt-debs && cd dist/apt-debs+ gh release download "$TAG" --repo "$REPO" --pattern checksums.txt+ for arch in amd64 arm64; do+ gh release download "$TAG" --repo "$REPO" --pattern "beacon_${VERSION}_linux_${arch}.deb"+ done+ # Every .deb must be listed; --ignore-missing alone would pass an unlisted one.+ for f in beacon_*.deb; do+ awk -v name="$f" '$2 == name { found = 1 } END { exit !found }' checksums.txt \+ || { echo "$f is not in checksums.txt" >&2; exit 1; }+ done+ sha256sum --check --ignore-missing --strict checksums.txt++ # Only the signed index is read back. An empty prefix is the first publish; a failed listing+ # fails the job rather than starting the repository over.+ - name: Fetch the published index+ if: steps.gate.outputs.publish == 'true'+ id: existing+ run: |+ set -euo pipefail+ # Counts Contents rather than reading KeyCount, which some S3-compatible stores omit; a+ # missing KeyCount read as zero would start the index over and drop every older version.+ # shellcheck disable=SC2016 # the backticks are a JMESPath literal, not a command+ count="$(aws s3api list-objects-v2 --bucket "$APT_S3_BUCKET" --prefix dists/stable/ --query 'length(Contents || `[]`)' --output text)"+ case "$count" in+ ''|*[!0-9]*) echo "unexpected object count '$count' for dists/stable/" >&2; exit 1 ;;+ esac+ if [ "$count" = "0" ]; then+ echo "::notice::the bucket has no published index; this is the first publish"+ echo "dir=" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ dir="$RUNNER_TEMP/apt-existing"+ aws s3 cp --no-progress "s3://$APT_S3_BUCKET/dists/stable/InRelease" "$dir/InRelease"+ for arch in amd64 arm64; do+ rel="main/binary-$arch/Packages"+ # An architecture added after the first publish has no index yet; build-apt-repo.sh+ # fails if InRelease lists one that is missing here.+ aws s3 cp --no-progress "s3://$APT_S3_BUCKET/dists/stable/$rel" "$dir/$rel" || true+ done+ echo "dir=$dir" >> "$GITHUB_OUTPUT"++ - name: Build and sign the repository update+ if: steps.gate.outputs.publish == 'true'+ env:+ FINGERPRINT: ${{ steps.key.outputs.fingerprint }}+ EXISTING: ${{ steps.existing.outputs.dir }}+ run: |+ set -euo pipefail+ set -- --out dist/apt-repo --keyring "$RUNNER_TEMP/beacon-archive-keyring.gpg" --signing-key "$FINGERPRINT"+ if [ -n "$EXISTING" ]; then+ set -- "$@" --existing "$EXISTING"+ fi+ sh packaging/linux/apt/build-apt-repo.sh "$@" dist/apt-debs/beacon_*.deb+ cp "$KEYRING_ASC" dist/apt-repo/beacon-archive-keyring.asc+ cp "$RUNNER_TEMP/beacon-archive-keyring.gpg" dist/apt-repo/beacon-archive-keyring.gpg++ - name: Check the update with apt before publishing+ if: steps.gate.outputs.publish == 'true'+ env:+ VERSION: ${{ steps.gate.outputs.version }}+ run: |+ set -euo pipefail+ # A rerun of a published tag stages no packages, so there is nothing local to download;+ # the check against the public URL below still runs.+ if [ -z "$(find dist/apt-repo/pool -name '*.deb')" ]; then+ echo "::notice::$TAG is already in the apt repository; only the index is republished"+ exit 0+ fi+ sh packaging/linux/apt/verify-apt-repo.sh "file:$PWD/dist/apt-repo" \+ "$RUNNER_TEMP/beacon-archive-keyring.gpg" "$VERSION"++ # Order matters: packages, then index files, then the signed Release files with InRelease+ # last, so no client is pointed at a file that is not there yet. Packages are immutable and+ # cached for a year; everything under dists/ and the keyring is revalidated on every request.+ - name: Publish to the bucket+ if: steps.gate.outputs.publish == 'true'+ working-directory: dist/apt-repo+ run: |+ set -euo pipefail+ bucket="s3://$APT_S3_BUCKET"+ if [ -d pool ]; then+ aws s3 cp --no-progress --recursive pool "$bucket/pool" \+ --content-type application/vnd.debian.binary-package \+ --cache-control 'public, max-age=31536000, immutable'+ fi+ aws s3 cp --no-progress --recursive dists/stable/main "$bucket/dists/stable/main" \+ --cache-control 'no-cache'+ for f in beacon-archive-keyring.asc beacon-archive-keyring.gpg; do+ aws s3 cp --no-progress "$f" "$bucket/$f" --cache-control 'no-cache'+ done+ for f in Release.gpg Release InRelease; do+ aws s3 cp --no-progress "dists/stable/$f" "$bucket/dists/stable/$f" \+ --content-type text/plain --cache-control 'no-cache'+ done++ - name: Check the published repository with apt+ if: steps.gate.outputs.publish == 'true'+ env:+ VERSION: ${{ steps.gate.outputs.version }}+ run: |+ set -euo pipefail+ sh packaging/linux/apt/verify-apt-repo.sh "${APT_REPO_URL%/}" \+ "$RUNNER_TEMP/beacon-archive-keyring.gpg" "$VERSION"+# One checksums.txt for everything a user installs, and signed provenance for all of it.## goreleaser writes checksums.txt before the .pkg and .msi exist, so they are appended here from thediff --git a/CLAUDE.md b/CLAUDE.mdindex ee71aeb7..8ffd0fd9 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -271,7 +271,7 @@ local GoReleaser publish only as a fallback when CI release automation is notavailable or the maintainer explicitly asks for a local release.A pushed `v*` tag runs the full release end-to-end via-`.github/workflows/release.yml` (no local publish steps required), in four jobs:+`.github/workflows/release.yml` (no local publish steps required), in five jobs:1. `goreleaser` (ubuntu, `environment: release`) builds the collector dists,publishes the GitHub release + changelog and tarballs, and updates the@@ -287,7 +287,19 @@ A pushed `v*` tag runs the full release end-to-end viaworks, and attaches the `.msi` and its `.sha256`. Deliberately **not** in the`release` environment: that environment gates signing secrets, and the MSI isunsigned, so this job has none to gate.-4. `finalize` (ubuntu, `needs` all three, runs whenever `goreleaser` succeeded)+4. `apt-repository` (ubuntu, `needs: goreleaser`, `environment: release`) adds the+ released `.deb` files, checked against `checksums.txt`, to the signed apt repository+ (`deb <APT_REPO_URL> stable main`) in an S3-compatible bucket, because 85-90 MB packages+ outgrow a git-backed Pages repo. `packaging/linux/apt/build-apt-repo.sh` reads back only the+ published index, verifies it against the committed public key+ `packaging/linux/apt/beacon-archive-keyring.asc` before re-signing (so a bucket edit is never+ laundered into a valid signature), refuses to replace a published version, and fails unless+ the secret's key signs for the committed one, so rotating the key is a reviewed commit.+ `verify-apt-repo.sh` runs a real, isolated `apt-get update`/`download` against the staged+ tree and again against the public URL; CI's `apt-repository` job runs `test-apt-repo.sh`.+ Upload order is pool, index, then `Release.gpg`/`Release`/`InRelease` with `Acquire-By-Hash`.+ Skipped with a notice until every `APT_*` setting exists, and for prerelease (`-`) tags.+5. `finalize` (ubuntu, `needs` goreleaser, package and windows-package, runs whenever `goreleaser` succeeded)appends the `.pkg`, `.msi` and `install.sh` to `checksums.txt` from the released bytes andrecords a GitHub build-provenance attestation (Sigstore, keyless, through theworkflow's OIDC token) for every file `checksums.txt` lists. It needs@@ -304,6 +316,9 @@ Required `release`-environment secrets: `HOMEBREW_TAP_TOKEN`,`DEVELOPER_ID_INSTALLER_CERT_P12`, `DEVELOPER_ID_INSTALLER_CERT_PASSWORD`,`NOTARY_API_KEY_P8`, `NOTARY_API_KEY_ID`, `NOTARY_API_ISSUER`; plus Variables`DEVELOPER_ID_APP_IDENTITY`, `DEVELOPER_ID_INSTALLER_IDENTITY`, `APPLE_TEAM_ID`.+The apt repository adds secrets `APT_GPG_PRIVATE_KEY` (armored, no passphrase),+`APT_S3_ACCESS_KEY_ID`, `APT_S3_SECRET_ACCESS_KEY` and Variables `APT_S3_BUCKET`,+`APT_REPO_URL` (https), plus optional `APT_S3_ENDPOINT` (R2/GCS) and `APT_S3_REGION`.Keep one-time Apple-side onboarding steps out of the repo and share them withthe release maintainer directly. Never use `pull_request_target` or builduntrusted PR code in the signing job.diff --git a/packaging/linux/apt/build-apt-repo.sh b/packaging/linux/apt/build-apt-repo.shnew file mode 100755index 00000000..800a9471--- /dev/null+++ b/packaging/linux/apt/build-apt-repo.sh@@ -0,0 +1,243 @@+#!/bin/sh+# Builds the next state of Beacon's signed apt repository from the .deb files of one release.+#+# packaging/linux/apt/build-apt-repo.sh --out DIR --keyring FILE --signing-key FPR \+# [--existing DIR] DEB...+#+# The published repository lives in object storage and keeps every version's .deb under pool/, so+# this script never downloads the pool. It takes the published index instead (--existing: a copy+# of dists/stable holding InRelease and main/binary-<arch>/Packages), checks that index against+# the committed public key (--keyring, a binary keyring), adds the new packages to it, and signs+# the result with --signing-key. Without --existing it starts an empty repository, which is right+# only for the very first publish.+#+# Checking the published index first matters because the result is re-signed: a Packages file+# altered in the bucket would otherwise come back out carrying a valid signature.+#+# DIR receives only what changed and must be uploaded in this order, so a client never reads an+# index that names a file not yet there:+# 1. pool/ the new .deb files (immutable; never overwritten)+# 2. dists/stable/main/ Packages, Packages.gz and their by-hash/ copies+# 3. dists/stable/Release.gpg, Release, then InRelease last+# The Release file sets Acquire-By-Hash, so a client that read the old InRelease mid-upload still+# fetches the index files it names through by-hash/ paths that are never replaced.+#+# A version already in the index is accepted again only with identical bytes, so rerunning a+# release is a no-op and a rebuilt package can never replace one that clients have cached.+#+# Needs apt-ftparchive (apt-utils), dpkg-deb, gpg and gpgv. Set APT_GPG_PASSPHRASE_FILE when the+# signing key has a passphrase.+set -eu++SUITE=stable+COMPONENT=main+ARCHES="amd64 arm64"++fail() {+ echo "build-apt-repo: $*" >&2+ exit 1+}++out=""+keyring=""+signing_key=""+existing=""+while [ $# -gt 0 ]; do+ case "$1" in+ --out) out="$2"; shift 2 ;;+ --keyring) keyring="$2"; shift 2 ;;+ --signing-key) signing_key="$2"; shift 2 ;;+ --existing) existing="$2"; shift 2 ;;+ --) shift; break ;;+ -*) fail "unknown option $1" ;;+ *) break ;;+ esac+done+[ -n "$out" ] || fail "--out is required"+[ -n "$keyring" ] || fail "--keyring is required"+[ -n "$signing_key" ] || fail "--signing-key is required"+[ $# -gt 0 ] || fail "no .deb files given"+[ -f "$keyring" ] || fail "keyring $keyring does not exist"+for tool in apt-ftparchive dpkg-deb gpg gpgv sha256sum; do+ command -v "$tool" >/dev/null 2>&1 || fail "$tool is not installed"+done+if [ -e "$out" ] && [ -n "$(ls -A "$out" 2>/dev/null)" ]; then+ fail "$out is not empty"+fi+mkdir -p "$out"+out="$(CDPATH='' cd -- "$out" && pwd)"+keyring="$(CDPATH='' cd -- "$(dirname -- "$keyring")" && pwd)/$(basename -- "$keyring")"++work="$(mktemp -d)"+trap 'rm -rf "$work"' EXIT INT TERM+dists="$out/dists/$SUITE"++# Prints one "package|version|arch sha256" line per stanza of a Packages file.+stanza_keys() {+ awk 'BEGIN { RS = "" }+ {+ p = v = a = s = ""+ n = split($0, line, "\n")+ for (i = 1; i <= n; i++) {+ if (line[i] ~ /^Package: /) p = substr(line[i], 10)+ else if (line[i] ~ /^Version: /) v = substr(line[i], 10)+ else if (line[i] ~ /^Architecture: /) a = substr(line[i], 15)+ else if (line[i] ~ /^SHA256: /) s = substr(line[i], 9)+ }+ print p "|" v "|" a " " s+ }' "$1"+}++# --- Check the published index -------------------------------------------------------------+for arch in $ARCHES; do+ : > "$work/old-$arch"+done+if [ -n "$existing" ]; then+ [ -f "$existing/InRelease" ] || fail "$existing/InRelease does not exist"+ gpgv --keyring "$keyring" --output "$work/old-Release" "$existing/InRelease" 2>"$work/gpgv.log" || {+ cat "$work/gpgv.log" >&2+ fail "the published InRelease is not signed by the committed key"+ }+ for arch in $ARCHES; do+ rel="$COMPONENT/binary-$arch/Packages"+ # The SHA256 block lists " <hash> <size> <path>"; only that block is read.+ want="$(awk -v rel="$rel" '+ /^[^ ]/ { in_sha = ($0 == "SHA256:") ; next }+ in_sha && $3 == rel { print $1 }' "$work/old-Release")"+ if [ -z "$want" ]; then+ # An architecture added after the first publish has no index yet.+ continue+ fi+ [ -f "$existing/$rel" ] || fail "the published InRelease lists $rel but it was not provided"+ got="$(sha256sum "$existing/$rel" | awk '{ print $1 }')"+ [ "$got" = "$want" ] || fail "the published $rel does not match its InRelease checksum"+ cp "$existing/$rel" "$work/old-$arch"+ done+fi++# --- Stage the new packages ----------------------------------------------------------------+stage="$work/stage"+mkdir -p "$stage"+for deb in "$@"; do+ [ -f "$deb" ] || fail "$deb does not exist"+ pkg="$(dpkg-deb -f "$deb" Package)"+ ver="$(dpkg-deb -f "$deb" Version)"+ arch="$(dpkg-deb -f "$deb" Architecture)"+ case " $ARCHES " in+ *" $arch "*) ;;+ *) fail "$deb is for $arch, which this repository does not carry" ;;+ esac+ first="$(printf '%s' "$pkg" | cut -c1)"+ dest="pool/$COMPONENT/$first/$pkg/${pkg}_${ver}_${arch}.deb"+ [ -e "$stage/$dest" ] && fail "two packages map to $dest"+ mkdir -p "$stage/$(dirname -- "$dest")"+ cp "$deb" "$stage/$dest"+done++for arch in $ARCHES; do+ (cd "$stage" && apt-ftparchive --arch "$arch" packages pool) > "$work/new-$arch"+ # apt-ftparchive includes Architecture: all packages in every arch; there are none here, but+ # a stanza for another arch would mean the staging above is wrong.+ stanza_keys "$work/new-$arch" | while read -r key _; do+ case "$key" in+ *"|$arch") ;;+ *) fail "unexpected stanza $key in the $arch index" ;;+ esac+ done+done++# --- Merge ---------------------------------------------------------------------------------+# A stanza already published under the same package, version and arch is kept as published when+# the bytes are identical and refused when they differ. Everything else is appended.+mkdir -p "$out/pool"+for arch in $ARCHES; do+ stanza_keys "$work/old-$arch" > "$work/old-keys-$arch"+ stanza_keys "$work/new-$arch" > "$work/new-keys-$arch"+ : > "$work/skip-$arch"+ while read -r key sha; do+ published="$(awk -v key="$key" '$1 == key { print $2 }' "$work/old-keys-$arch")"+ if [ -z "$published" ]; then+ continue+ fi+ [ "$published" = "$sha" ] || fail "$key is already published with different contents (published $published, new $sha); a released version cannot be replaced"+ echo "$key" >> "$work/skip-$arch"+ done < "$work/new-keys-$arch"++ index_dir="$dists/$COMPONENT/binary-$arch"+ mkdir -p "$index_dir"+ {+ if [ -s "$work/old-$arch" ]; then+ awk 'BEGIN { RS = ""; ORS = "\n\n" } { print }' "$work/old-$arch"+ fi+ awk -v skipfile="$work/skip-$arch" '+ BEGIN {+ while ((getline k < skipfile) > 0) skip[k] = 1+ RS = ""; ORS = "\n\n"+ }+ {+ p = v = a = ""+ n = split($0, line, "\n")+ for (i = 1; i <= n; i++) {+ if (line[i] ~ /^Package: /) p = substr(line[i], 10)+ else if (line[i] ~ /^Version: /) v = substr(line[i], 10)+ else if (line[i] ~ /^Architecture: /) a = substr(line[i], 15)+ }+ if (!((p "|" v "|" a) in skip)) print+ }' "$work/new-$arch"+ } > "$index_dir/Packages"+ gzip -9n < "$index_dir/Packages" > "$index_dir/Packages.gz"++ # Only packages not already published are uploaded.+ while read -r key _; do+ grep -qxF "$key" "$work/skip-$arch" && continue+ pkg="${key%%|*}"+ rest="${key#*|}"+ ver="${rest%%|*}"+ first="$(printf '%s' "$pkg" | cut -c1)"+ rel="pool/$COMPONENT/$first/$pkg/${pkg}_${ver}_${arch}.deb"+ mkdir -p "$out/$(dirname -- "$rel")"+ cp "$stage/$rel" "$out/$rel"+ done < "$work/new-keys-$arch"+done++# --- Release, signed ----------------------------------------------------------------------+# Generated before the by-hash copies exist, so it lists only the index files.+(cd "$dists" && apt-ftparchive \+ -o APT::FTPArchive::Release::Origin=Beacon \+ -o APT::FTPArchive::Release::Label=Beacon \+ -o APT::FTPArchive::Release::Suite="$SUITE" \+ -o APT::FTPArchive::Release::Codename="$SUITE" \+ -o APT::FTPArchive::Release::Architectures="$ARCHES" \+ -o APT::FTPArchive::Release::Components="$COMPONENT" \+ -o APT::FTPArchive::Release::Description="Beacon Endpoint Agent" \+ -o APT::FTPArchive::Release::Acquire-By-Hash=yes \+ release .) > "$work/Release"+mv "$work/Release" "$dists/Release"++for arch in $ARCHES; do+ index_dir="$dists/$COMPONENT/binary-$arch"+ mkdir -p "$index_dir/by-hash/SHA256"+ for f in Packages Packages.gz; do+ sum="$(sha256sum "$index_dir/$f" | awk '{ print $1 }')"+ cp "$index_dir/$f" "$index_dir/by-hash/SHA256/$sum"+ done+done++set -- --batch --yes --local-user "$signing_key"+if [ -n "${APT_GPG_PASSPHRASE_FILE:-}" ]; then+ set -- "$@" --pinentry-mode loopback --passphrase-file "$APT_GPG_PASSPHRASE_FILE"+fi+gpg "$@" --digest-algo SHA512 --clearsign --output "$dists/InRelease" "$dists/Release"+gpg "$@" --digest-algo SHA512 --armor --detach-sign --output "$dists/Release.gpg" "$dists/Release"++# The signing key in the secret must be the key users were told to trust.+gpgv --keyring "$keyring" "$dists/InRelease" 2>"$work/gpgv.log" || {+ cat "$work/gpgv.log" >&2+ fail "the new InRelease does not verify against $keyring; the signing key is not the committed key"+}+gpgv --keyring "$keyring" "$dists/Release.gpg" "$dists/Release" 2>"$work/gpgv.log" || {+ cat "$work/gpgv.log" >&2+ fail "the new Release.gpg does not verify against $keyring"+}++echo "build-apt-repo: staged $(find "$out/pool" -name '*.deb' | wc -l | tr -d ' ') new package(s) in $out"diff --git a/packaging/linux/apt/test-apt-repo.sh b/packaging/linux/apt/test-apt-repo.shnew file mode 100755index 00000000..2fadb218--- /dev/null+++ b/packaging/linux/apt/test-apt-repo.sh@@ -0,0 +1,157 @@+#!/bin/sh+# Exercises build-apt-repo.sh and verify-apt-repo.sh against stand-in packages, a throwaway+# signing key and a local directory standing in for the bucket. Runs in CI on ubuntu; needs+# apt-utils, dpkg-dev and gnupg, and no network.+#+# packaging/linux/apt/test-apt-repo.sh+#+# It publishes two releases in sequence and checks that both stay installable, that a rerun is a+# no-op, and that a replaced package, a tampered index and the wrong signing key are all refused.+set -eu++HERE="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"+BUILD="$HERE/build-apt-repo.sh"+VERIFY="$HERE/verify-apt-repo.sh"++work="$(mktemp -d)"+trap 'rm -rf "$work"' EXIT INT TERM+export GNUPGHOME="$work/gnupg"+mkdir -m 0700 "$GNUPGHOME"++pass() { echo "ok - $*"; }+fail() { echo "FAIL - $*" >&2; exit 1; }++new_key() {+ gpg --batch --quiet --pinentry-mode loopback --passphrase '' \+ --quick-generate-key "$1" ed25519 sign never+ gpg --batch --with-colons --list-keys "$1" | awk -F: '$1 == "fpr" { print $10; exit }'+}++# make_deb VERSION ARCH [EXTRA] -- EXTRA changes the bytes without changing the version.+make_deb() {+ root="$work/debroot-$1-$2"+ rm -rf "$root"+ mkdir -p "$root/DEBIAN" "$root/opt/beacon"+ cat > "$root/DEBIAN/control" <<EOF+Package: beacon+Version: $1+Architecture: $2+Maintainer: Beacon Test <[email protected]>+Description: stand-in package for the apt repository test+EOF+ printf '%s %s %s\n' "$1" "$2" "${3:-}" > "$root/opt/beacon/marker"+ mkdir -p "$work/debs"+ dpkg-deb --root-owner-group --build "$root" "$work/debs/beacon_$1_linux_$2.deb" >/dev/null+ echo "$work/debs/beacon_$1_linux_$2.deb"+}++# publish OUT -- copies a staged tree into the bucket in the documented order.+publish() {+ mkdir -p "$work/bucket"+ if [ -d "$1/pool" ]; then+ cp -R "$1/pool" "$work/bucket/"+ fi+ mkdir -p "$work/bucket/dists/stable"+ cp -R "$1/dists/stable/main" "$work/bucket/dists/stable/"+ for f in Release.gpg Release InRelease; do+ cp "$1/dists/stable/$f" "$work/bucket/dists/stable/$f"+ done+}++# existing -- the copy of the published index the release workflow downloads.+existing() {+ rm -rf "$work/existing"+ mkdir -p "$work/existing/main"+ cp "$work/bucket/dists/stable/InRelease" "$work/existing/"+ for arch in amd64 arm64; do+ mkdir -p "$work/existing/main/binary-$arch"+ cp "$work/bucket/dists/stable/main/binary-$arch/Packages" "$work/existing/main/binary-$arch/"+ done+}++fpr="$(new_key 'Beacon Test Archive <[email protected]>')"+gpg --batch --export "$fpr" > "$work/keyring.gpg"+other_fpr="$(new_key 'Someone Else <[email protected]>')"+gpg --batch --export "$other_fpr" > "$work/other-keyring.gpg"++# 1. First publish.+deb_a1="$(make_deb 1.0.0 amd64)"+deb_b1="$(make_deb 1.0.0 arm64)"+sh "$BUILD" --out "$work/out1" --keyring "$work/keyring.gpg" --signing-key "$fpr" "$deb_a1" "$deb_b1"+sh "$VERIFY" "file:$work/out1" "$work/keyring.gpg" 1.0.0+publish "$work/out1"+sh "$VERIFY" "file:$work/bucket" "$work/keyring.gpg" 1.0.0+pass "first publish serves 1.0.0"++# 2. Second release, merged onto the published index.+existing+deb_a2="$(make_deb 1.0.1 amd64)"+deb_b2="$(make_deb 1.0.1 arm64)"+sh "$BUILD" --out "$work/out2" --keyring "$work/keyring.gpg" --signing-key "$fpr" \+ --existing "$work/existing" "$deb_a2" "$deb_b2"+[ "$(find "$work/out2/pool" -name '*.deb' | wc -l | tr -d ' ')" = 2 ] || fail "second publish should stage only the two new packages"+publish "$work/out2"+sh "$VERIFY" "file:$work/bucket" "$work/keyring.gpg" 1.0.1+sh "$VERIFY" "file:$work/bucket" "$work/keyring.gpg" 1.0.0+pass "second publish serves 1.0.1 and keeps 1.0.0"++grep -q '^Acquire-By-Hash: yes$' "$work/bucket/dists/stable/Release" || fail "Release does not set Acquire-By-Hash"+sum="$(sha256sum "$work/bucket/dists/stable/main/binary-amd64/Packages" | awk '{ print $1 }')"+[ -f "$work/bucket/dists/stable/main/binary-amd64/by-hash/SHA256/$sum" ] || fail "no by-hash copy of the amd64 index"+pass "Release sets Acquire-By-Hash and the by-hash copies exist"++# 3. Rerunning the same release changes nothing.+existing+sh "$BUILD" --out "$work/out3" --keyring "$work/keyring.gpg" --signing-key "$fpr" \+ --existing "$work/existing" "$deb_a2" "$deb_b2"+[ -z "$(find "$work/out3/pool" -name '*.deb')" ] || fail "a rerun staged packages again"+cmp -s "$work/out3/dists/stable/main/binary-amd64/Packages" "$work/bucket/dists/stable/main/binary-amd64/Packages" \+ || fail "a rerun changed the amd64 index"+[ "$(grep -c '^Package: ' "$work/out3/dists/stable/main/binary-amd64/Packages")" = 2 ] || fail "the amd64 index should list two versions"+pass "a rerun is a no-op"++# 4. A rebuilt package cannot replace a published version.+deb_a2_changed="$(make_deb 1.0.1 amd64 rebuilt)"+if sh "$BUILD" --out "$work/out4" --keyring "$work/keyring.gpg" --signing-key "$fpr" \+ --existing "$work/existing" "$deb_a2_changed" 2>"$work/err4"; then+ fail "a changed 1.0.1 was accepted"+fi+grep -q 'already published with different contents' "$work/err4" || { cat "$work/err4" >&2; fail "wrong error for a changed package"; }+pass "a replaced package is refused"++# 5. A tampered index in the bucket is refused rather than re-signed.+existing+printf '\nPackage: evil\nVersion: 9\nArchitecture: amd64\nFilename: pool/evil.deb\n' >> "$work/existing/main/binary-amd64/Packages"+deb_a3="$(make_deb 1.0.2 amd64)"+deb_b3="$(make_deb 1.0.2 arm64)"+if sh "$BUILD" --out "$work/out5" --keyring "$work/keyring.gpg" --signing-key "$fpr" \+ --existing "$work/existing" "$deb_a3" "$deb_b3" 2>"$work/err5"; then+ fail "a tampered index was accepted"+fi+grep -q 'does not match its InRelease checksum' "$work/err5" || { cat "$work/err5" >&2; fail "wrong error for a tampered index"; }+pass "a tampered index is refused"++# 6. A published index signed by another key is refused.+existing+if sh "$BUILD" --out "$work/out6" --keyring "$work/other-keyring.gpg" --signing-key "$other_fpr" \+ --existing "$work/existing" "$deb_a3" "$deb_b3" 2>"$work/err6"; then+ fail "an index signed by another key was accepted"+fi+grep -q 'not signed by the committed key' "$work/err6" || { cat "$work/err6" >&2; fail "wrong error for another key's index"; }+pass "an index signed by another key is refused"++# 7. A signing key that is not the committed key is refused before anything is published.+if sh "$BUILD" --out "$work/out7" --keyring "$work/keyring.gpg" --signing-key "$other_fpr" \+ --existing "$work/existing" "$deb_a3" "$deb_b3" 2>"$work/err7"; then+ fail "signing with another key was accepted"+fi+grep -q 'signing key is not the committed key' "$work/err7" || { cat "$work/err7" >&2; fail "wrong error for the wrong signing key"; }+pass "the wrong signing key is refused"++# 8. A client that trusts another key refuses the repository.+if sh "$VERIFY" "file:$work/bucket" "$work/other-keyring.gpg" 1.0.1 >/dev/null 2>&1; then+ fail "a client trusting another key accepted the repository"+fi+pass "a client trusting another key refuses the repository"++echo "all apt repository checks passed"diff --git a/packaging/linux/apt/verify-apt-repo.sh b/packaging/linux/apt/verify-apt-repo.shnew file mode 100755index 00000000..a8eb691c--- /dev/null+++ b/packaging/linux/apt/verify-apt-repo.sh@@ -0,0 +1,76 @@+#!/bin/sh+# Checks a Beacon apt repository the way a client sees it: runs a real apt-get update against it+# with only the given keyring trusted, then downloads one package per architecture and checks the+# version it carries.+#+# packaging/linux/apt/verify-apt-repo.sh URL KEYRING VERSION [PACKAGE]+#+# URL is the repository root (https://..., or file:/path for a staged copy). KEYRING is a binary+# keyring file. Runs as an ordinary user: apt gets private state, cache and source directories and+# never reads or changes the host's own configuration.+#+# apt-get update runs with --error-on=any, so a signature problem, a checksum mismatch or a+# missing index fails here instead of becoming a warning.+set -eu++fail() {+ echo "verify-apt-repo: $*" >&2+ exit 1+}++[ $# -ge 3 ] || fail "usage: verify-apt-repo.sh URL KEYRING VERSION [PACKAGE]"+url="$1"+keyring="$2"+version="$3"+package="${4:-beacon}"+ARCHES="amd64 arm64"++[ -f "$keyring" ] || fail "keyring $keyring does not exist"+keyring="$(CDPATH='' cd -- "$(dirname -- "$keyring")" && pwd)/$(basename -- "$keyring")"++dir="$(mktemp -d)"+trap 'rm -rf "$dir"' EXIT INT TERM+mkdir -p "$dir/lists/partial" "$dir/cache/archives/partial" "$dir/sources.list.d" \+ "$dir/apt.conf.d" "$dir/preferences.d" "$dir/trusted.gpg.d" "$dir/download"+: > "$dir/status"+# The only source and the only trusted key.+echo "deb [signed-by=$keyring arch=amd64,arm64] $url stable main" > "$dir/sources.list"++# A private configuration in place of /etc/apt/apt.conf. Dir::Etc::Parts names an empty+# directory, so none of the host's apt.conf.d snippets (hooks, proxies, Docker's clean hook) apply.+cat > "$dir/apt.conf" <<EOF+Dir::State::Lists "$dir/lists";+Dir::State::status "$dir/status";+Dir::Cache "$dir/cache";+Dir::Etc::SourceList "$dir/sources.list";+Dir::Etc::SourceParts "$dir/sources.list.d";+Dir::Etc::Parts "$dir/apt.conf.d";+Dir::Etc::PreferencesParts "$dir/preferences.d";+Dir::Etc::Trusted "/dev/null";+Dir::Etc::TrustedParts "$dir/trusted.gpg.d";+Debug::NoLocking "1";+APT::Sandbox::User "$(id -un)";+APT::Architecture "amd64";+APT::Architectures { "amd64"; "arm64"; };+Acquire::Retries "3";+EOF+APT_CONFIG="$dir/apt.conf"+export APT_CONFIG++apt-get --error-on=any update || fail "apt-get update against $url failed"++for arch in $ARCHES; do+ (cd "$dir/download" && apt-get download "$package:$arch=$version") \+ || fail "could not download $package:$arch=$version from $url"+done++for arch in $ARCHES; do+ deb="$(find "$dir/download" -name "${package}_*_${arch}.deb" | head -n 1)"+ [ -n "$deb" ] || fail "apt downloaded no $arch package"+ got_version="$(dpkg-deb -f "$deb" Version)"+ got_arch="$(dpkg-deb -f "$deb" Architecture)"+ [ "$got_version" = "$version" ] || fail "$arch package reports version $got_version, expected $version"+ [ "$got_arch" = "$arch" ] || fail "$arch package reports architecture $got_arch"+done++echo "verify-apt-repo: $url serves $package $version for $ARCHES, signed by the given key"