Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

ci: publish nightly wheels to a rolling nightly release on this repo

Xiaoze Fan7dfc37addd54
diff --git a/.github/workflows/nightly-wheels.yml b/.github/workflows/nightly-wheels.yml
index 5cf55434..31a3424f 100644
--- a/.github/workflows/nightly-wheels.yml
+++ b/.github/workflows/nightly-wheels.yml
@@ -1,5 +1,8 @@
-# Nightly engine wheels: build on the self-hosted EC2 node and publish to the
-# FreeToken-Web rolling `beta` release.
+# Nightly engine wheels: build on the self-hosted EC2 node and publish to this repo's
+# rolling `nightly` prerelease. scripts/publish-wheels.sh moves the `nightly` tag to the
+# built commit after every publish. Promotion to the curated FreeToken-Web `beta`
+# channel is a separate, hand-triggered lane (promote-beta.yml) that copies the pair
+# without rebuilding.
#
# The node currently runs ALWAYS-ON, so this workflow does no power management.
# Reintroduce the elastic variant (OIDC start-node/stop-node jobs around the build,
@@ -16,8 +19,7 @@
# nightly is fully identified by the commit it was built from. Nights where HEAD is
# already published are skipped before the EC2 node is ever started.
#
-# Formal (tagged) releases are out of scope here for now -- this workflow is the
-# nightly channel only.
+# Formal (tagged) releases are out of scope here -- see release.yml.
name: Nightly wheels
@@ -40,38 +42,41 @@ concurrency:
group: nightly-wheels
env:
- WEB_REPO: FlashML-org/FreeToken-Web
- WEB_TAG: beta
+ RELEASE_TAG: nightly
jobs:
# Decide whether HEAD needs building at all. Runs on a hosted runner so a no-op
# night costs nothing and never touches AWS.
check:
- # Guard against runs from forks of this repo; update on an org transfer.
- if: github.repository == 'FlashML-org/FreeToken'
+ # Guard against runs from forks of this repo; update on an org transfer. Only main
+ # feeds the nightly channel: a dispatch on any other branch is a no-op.
+ if: github.repository == 'FlashML-org/FreeToken' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
build: ${{ steps.decide.outputs.build }}
steps:
- - name: Compare HEAD against the published beta stamp
+ - name: Compare HEAD against the published nightly stamp
id: decide
env:
GH_TOKEN: ${{ github.token }}
FORCE: ${{ inputs.force }}
run: |
head_stamp="${GITHUB_SHA:0:9}"
- # The release also carries win_amd64 wheels with their own stamp; this
- # workflow only builds linux, so read the linux manifest that
- # scripts/publish-wheels.sh writes. Before the first manifest exists, fall
- # back to the stamp in the linux runtime wheel's name.
+ # Read the manifest scripts/publish-wheels.sh writes. Before the first manifest
+ # exists, fall back to the stamp in the runtime wheel's name.
published_stamp="$(curl -fsSL \
- "https://github.com/$WEB_REPO/releases/download/$WEB_TAG/engine-linux_x86_64.json" \
+ "https://github.com/$GITHUB_REPOSITORY/releases/download/$RELEASE_TAG/engine-linux_x86_64.json" \
2>/dev/null | jq -r '.commit // empty' || true)"
- [ -n "$published_stamp" ] || published_stamp="$(gh api "repos/$WEB_REPO/releases/tags/$WEB_TAG" \
+ [ -n "$published_stamp" ] || published_stamp="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" \
--jq '.assets[].name' 2>/dev/null \
| grep -E '^freetoken-.*linux_x86_64\.whl$' \
| grep -oE '\+g[0-9a-f]{7,}' | head -1 | sed 's/^+g//' || true)"
+ # The tag is moved last; if it lags the manifest, a publish died half-way and
+ # must run again.
+ tag_sha="$(gh api "repos/$GITHUB_REPOSITORY/git/refs/tags/$RELEASE_TAG" \
+ --jq '.object.sha' 2>/dev/null || true)"
+ [ "${tag_sha:0:9}" = "$published_stamp" ] || published_stamp=""
echo "HEAD: $head_stamp published: ${published_stamp:-<none>}"
if [ "$FORCE" = "true" ] || [ "$head_stamp" != "$published_stamp" ]; then
echo "build=true" >> "$GITHUB_OUTPUT"
@@ -102,23 +107,33 @@ jobs:
if-no-files-found: error
retention-days: 7
- # Publish from a hosted runner: the cross-repo token is never present on the
- # self-hosted node, so a compromised build node cannot reach the release channel.
+ # Publish from a hosted runner with the job-scoped GITHUB_TOKEN. The self-hosted
+ # node never holds a token that can write releases, so a compromised build node
+ # cannot reach the nightly channel. The job-level group is shared with
+ # promote-beta.yml's fetch job, so a promotion never reads the release between
+ # the prune and the upload.
publish:
needs: [check, build]
if: needs.check.outputs.build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
- environment: release
+ concurrency:
+ group: nightly-release-assets
+ permissions:
+ contents: write
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ persist-credentials: false
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: wheels
path: dist
- - name: Publish to the beta release
+ - name: Publish to the nightly release
env:
- GH_TOKEN: ${{ secrets.FREETOKEN_WEB_TOKEN }}
- FREETOKEN_WEB_REPO: ${{ env.WEB_REPO }}
- FREETOKEN_WEB_TAG: ${{ env.WEB_TAG }}
+ GH_TOKEN: ${{ github.token }}
+ FREETOKEN_RELEASE_REPO: ${{ github.repository }}
+ FREETOKEN_RELEASE_TAG: ${{ env.RELEASE_TAG }}
+ FREETOKEN_RELEASE_ROLLING: "1"
+ FREETOKEN_RELEASE_COMMIT: ${{ github.sha }}
run: scripts/publish-wheels.sh dist
diff --git a/.github/workflows/promote-beta.yml b/.github/workflows/promote-beta.yml
new file mode 100644
index 00000000..8e964676
--- /dev/null
+++ b/.github/workflows/promote-beta.yml
@@ -0,0 +1,99 @@
+# Promote the nightly wheel pair to FreeToken-Web's curated `beta` release.
+#
+# Nothing is built here: the pair is copied byte-for-byte from this repo's rolling
+# `nightly` release (or from one nightly-wheels run's `wheels` artifact, kept 7 days),
+# so beta only ever carries wheels that nightly already published. Linux only -- the
+# win_amd64 assets on beta are published by hand from FreeToken-Windows and are never
+# touched; scripts/publish-wheels.sh prunes per platform.
+#
+# Trigger policy: workflow_dispatch only. GitHub requires write permission to dispatch,
+# and this is the only lane that holds the cross-repo FreeToken-Web token (an
+# ENVIRONMENT secret on `release`). Nothing here runs on the self-hosted node.
+#
+# The `release` environment carries the PAT with no protection rules, so write access
+# to dispatch is the gate. Add required reviewers there if that is ever too little.
+#
+# To promote a commit nightly has not built yet, run nightly-wheels.yml with
+# force=true first, then this workflow.
+
+name: Promote nightly to beta
+
+on:
+ workflow_dispatch:
+ inputs:
+ expect_commit:
+ description: "Commit (7-40 hex) the pair must be built from; empty = whatever nightly holds"
+ type: string
+ default: ""
+ run_id:
+ description: "Take the pair from this nightly-wheels run's artifact instead of the nightly release"
+ type: string
+ default: ""
+
+permissions:
+ contents: read
+
+env:
+ WEB_REPO: FlashML-org/FreeToken-Web
+ WEB_TAG: beta
+
+jobs:
+ # Reads the nightly release, so it shares a group with nightly-wheels.yml's publish
+ # job and never sees the window between prune and upload. Kept apart from the PAT
+ # job: a promotion waiting on the environment gate must not hold that group.
+ fetch:
+ # Guard against runs from forks of this repo; update on an org transfer.
+ if: github.repository == 'FlashML-org/FreeToken'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ concurrency:
+ group: nightly-release-assets
+ permissions:
+ contents: read
+ # gh run download of another run's artifact
+ actions: read
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ persist-credentials: false
+ - name: Fetch the nightly pair
+ env:
+ GH_TOKEN: ${{ github.token }}
+ NIGHTLY_REPO: ${{ github.repository }}
+ NIGHTLY_TAG: nightly
+ EXPECT_COMMIT: ${{ inputs.expect_commit }}
+ RUN_ID: ${{ inputs.run_id }}
+ run: scripts/ci/fetch-nightly-pair.sh dist
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: promote-wheels
+ path: dist/*.whl
+ if-no-files-found: error
+ retention-days: 1
+
+ publish:
+ needs: fetch
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ environment: release
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ persist-credentials: false
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: promote-wheels
+ path: dist
+ - name: Publish to the beta release
+ env:
+ GH_TOKEN: ${{ secrets.FREETOKEN_WEB_TOKEN }}
+ FREETOKEN_RELEASE_REPO: ${{ env.WEB_REPO }}
+ FREETOKEN_RELEASE_TAG: ${{ env.WEB_TAG }}
+ run: scripts/publish-wheels.sh dist
+ - name: Summary
+ run: |
+ {
+ echo "Promoted to https://github.com/$WEB_REPO/releases/tag/$WEB_TAG:"
+ echo
+ for w in dist/*.whl; do echo "- \`$(basename "$w")\`"; done
+ } >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 18dfe617..b6c75dc2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -8,9 +8,10 @@
# this in FREETOKEN_BUILD_RELEASE mode and refuses otherwise.
#
# This lane is separate from nightly-wheels.yml on purpose:
-# - nightly ships +g<sha>-stamped, linux_x86_64-tagged cp312 wheels to the
-# FreeToken-Web rolling `beta` release, which shipped Desktops resolve by
-# asset name -- that channel's naming must not change.
+# - nightly ships +g<sha>-stamped, linux_x86_64-tagged cp312 wheels to this repo's
+# rolling `nightly` prerelease, and promote-beta.yml copies that pair to the
+# FreeToken-Web `beta` release, which shipped Desktops resolve by asset name --
+# that naming must not change.
# - release ships bare-versioned, manylinux-retagged cp310-cp313 wheels, which
# is what PyPI accepts.
#
@@ -168,7 +169,7 @@ jobs:
# Upload-only draft release on THIS repo carrying every wheel, most importantly
# the kernel-cache one PyPI cannot host. Deliberately not scripts/publish-wheels.sh:
- # its prune-then-upload semantics and stamp-pairing gate are for the rolling beta
+ # its prune-then-upload semantics and stamp-pairing gate are for the rolling nightly
# channel, and would be wrong against an immutable version tag.
github-release:
needs: build
diff --git a/docs/install.md b/docs/install.md
index f5205ab3..9438c21a 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -23,6 +23,26 @@ uv venv && source .venv/bin/activate
uv pip install -e ".[accel]"
```
+## Method 3: Nightly wheels
+
+Every night `main` is built into a wheel pair on the rolling
+[`nightly` release](https://github.com/FlashML-org/FreeToken/releases/tag/nightly):
+the `freetoken` runtime (CPython 3.12, Linux x86_64) and the matching
+`freetoken-kernel-cache` with FreeToken's own CUDA kernels prebuilt.
+Install both from the URLs on that release page:
+
+```bash
+uv pip install \
+ "freetoken[accel] @ https://github.com/FlashML-org/FreeToken/releases/download/nightly/<runtime wheel>" \
+ "https://github.com/FlashML-org/FreeToken/releases/download/nightly/<kernel-cache wheel>"
+```
+
+Filenames carry a `+g<sha>` build stamp and change every night, and the `nightly`
+tag moves with them. Pin a wheel URL, never the tag. A local copy of the tag goes
+stale: `git fetch` leaves it alone and `git fetch --tags` refuses to overwrite it;
+refresh it with `git fetch --force origin tag nightly`. `engine-linux_x86_64.json`
+next to the wheels names the current pair for scripts.
+
## Verify
```bash
diff --git a/scripts/build-release-wheels.sh b/scripts/build-release-wheels.sh
index 27a5c437..f8ab773f 100755
--- a/scripts/build-release-wheels.sh
+++ b/scripts/build-release-wheels.sh
@@ -152,7 +152,8 @@ stamp_version() {
local version tag
version="$(sed -nE 's/^__version__ = "([^"+]+)".*$/\1/p' "$VERSION_FILE")"
[[ -n "$version" ]] || die "cannot read a version from $VERSION_FILE"
- tag="$(git -C "$ROOT" describe --exact-match --tags HEAD 2>/dev/null)" \
+ # --match: the rolling `nightly` tag can sit on the same commit as the release tag.
+ tag="$(git -C "$ROOT" describe --exact-match --tags --match 'v*' HEAD 2>/dev/null)" \
|| die "FREETOKEN_BUILD_RELEASE: HEAD is not at a tag (expected tag v$version)."
[[ "$tag" == "v$version" ]] \
|| die "FREETOKEN_BUILD_RELEASE: HEAD tag is '$tag' but version.py says '$version' (expected tag v$version)."
diff --git a/scripts/ci/fetch-nightly-pair.sh b/scripts/ci/fetch-nightly-pair.sh
new file mode 100755
index 00000000..bb807bfa
--- /dev/null
+++ b/scripts/ci/fetch-nightly-pair.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+#
+# Fetch the nightly linux wheel pair into a dist dir, ready for scripts/publish-wheels.sh.
+#
+# Usage:
+# scripts/ci/fetch-nightly-pair.sh [dist-dir] (default: ./dist)
+#
+# Source is the rolling nightly release (its engine-linux_x86_64.json names the pair), or
+# with RUN_ID the `wheels` artifact of one nightly-wheels run, which GitHub keeps for 7
+# days -- the way to promote an earlier night. Release downloads are checked against the
+# manifest's sha256 and size; every wheel's filename stamp is checked against the source
+# commit and, when given, EXPECT_COMMIT.
+#
+# Environment:
+# NIGHTLY_REPO repo holding the nightly release (default: FlashML-org/FreeToken)
+# NIGHTLY_TAG its tag (default: nightly)
+# RUN_ID nightly-wheels run id to take the artifact from instead
+# EXPECT_COMMIT sha prefix (7-40 hex) the pair must have been built from
+set -euo pipefail
+
+REPO="${NIGHTLY_REPO:-FlashML-org/FreeToken}"
+TAG="${NIGHTLY_TAG:-nightly}"
+RUN_ID="${RUN_ID:-}"
+EXPECT="${EXPECT_COMMIT:-}"
+DIST="${1:-dist}"
+
+say() { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
+die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
+
+for tool in gh jq curl sha256sum; do
+ command -v "$tool" >/dev/null 2>&1 || die "$tool not found"
+done
+if [ -n "$EXPECT" ]; then
+ [[ "$EXPECT" =~ ^[0-9a-f]{7,40}$ ]] || die "EXPECT_COMMIT must be 7-40 hex chars (got '$EXPECT')"
+fi
+# Stamps are 9 chars, EXPECT_COMMIT may be shorter or the full sha.
+same_commit() { [[ "$1" == "$2"* || "$2" == "$1"* ]]; }
+
+mkdir -p "$DIST"
+[ -z "$(ls -A "$DIST")" ] || die "$DIST is not empty"
+
+if [ -n "$RUN_ID" ]; then
+ [[ "$RUN_ID" =~ ^[0-9]+$ ]] || die "RUN_ID must be a plain number (got '$RUN_ID')"
+ run_json="$(gh api "repos/$REPO/actions/runs/$RUN_ID")" || die "cannot read run $RUN_ID on $REPO"
+ workflow="$(jq -r .name <<<"$run_json")"
+ branch="$(jq -r .head_branch <<<"$run_json")"
+ sha="$(jq -r .head_sha <<<"$run_json")"
+ conclusion="$(jq -r .conclusion <<<"$run_json")"
+ [ "$workflow" = "Nightly wheels" ] || die "run $RUN_ID is '$workflow', not a Nightly wheels run"
+ [ "$branch" = main ] || die "run $RUN_ID built branch '$branch', not main"
+ [ "$conclusion" = success ] || die "run $RUN_ID concluded '$conclusion', not success"
+ if [ -n "$EXPECT" ]; then
+ same_commit "$sha" "$EXPECT" || die "run $RUN_ID built ${sha:0:9}, not $EXPECT"
+ fi
+ # A night whose HEAD was already published succeeds without building, so a green
+ # run is not proof of an artifact; check for it and its 7-day expiry explicitly.
+ artifact="$(gh api "repos/$REPO/actions/runs/$RUN_ID/artifacts" \
+ --jq '.artifacts[] | select(.name == "wheels") | if .expired then "expired" else "ok" end')"
+ case "$artifact" in
+ ok) ;;
+ expired) die "run $RUN_ID's wheels artifact has expired (7-day retention)" ;;
+ *) die "run $RUN_ID has no wheels artifact -- a night whose HEAD was already published builds nothing" ;;
+ esac
+ say "downloading the 'wheels' artifact of run $RUN_ID (${sha:0:9})"
+ gh run download "$RUN_ID" -R "$REPO" -n wheels -D "$DIST"
+ source_commit="${sha:0:9}"
+else
+ url="https://github.com/$REPO/releases/download/$TAG/engine-linux_x86_64.json"
+ manifest="$(curl -fsSL "$url")" || die "cannot fetch $url"
+ source_commit="$(jq -r '.commit // empty' <<<"$manifest")"
+ [ -n "$source_commit" ] || die "the manifest at $url names no commit"
+ if [ -n "$EXPECT" ]; then
+ same_commit "$source_commit" "$EXPECT" || die "$TAG currently holds $source_commit, not $EXPECT"
+ fi
+ say "$TAG holds $source_commit (published $(jq -r .published <<<"$manifest"))"
+ # The manifest is data from the network; only a plain wheel filename served from
+ # this very release is acceptable, whatever the JSON says.
+ prefix="https://github.com/$REPO/releases/download/$TAG/"
+ for part in runtime kernel_cache; do
+ name="$(jq -r ".$part.name" <<<"$manifest")"
+ part_url="$(jq -r ".$part.url" <<<"$manifest")"
+ want_sha="$(jq -r ".$part.sha256" <<<"$manifest")"
+ want_size="$(jq -r ".$part.size" <<<"$manifest")"
+ [[ "$name" =~ ^freetoken[_-][A-Za-z0-9._+-]+\.whl$ ]] || die "manifest names a suspicious asset: '$name'"
+ [[ "$part_url" == "$prefix"* ]] || die "manifest URL is not under $prefix: $part_url"
+ [[ "$want_sha" =~ ^[0-9a-f]{64}$ && "$want_size" =~ ^[0-9]+$ ]] || die "manifest has a malformed sha256/size for $part"
+ say "downloading $name"
+ curl -fsSL -o "$DIST/$name" "$part_url" || die "download failed: $part_url"
+ got_size="$(wc -c <"$DIST/$name" | tr -d ' ')"
+ [ "$got_size" = "$want_size" ] || die "$name: size $got_size, manifest says $want_size"
+ got_sha="$(sha256sum "$DIST/$name" | cut -d' ' -f1)"
+ [ "$got_sha" = "$want_sha" ] || die "$name: sha256 does not match the manifest"
+ done
+fi
+
+# The filename stamp is what the publisher pairs and the consumers see; it must agree
+# with the source no matter how the wheels were fetched.
+shopt -s nullglob
+wheels=("$DIST"/*.whl)
+[ "${#wheels[@]}" -eq 2 ] || die "expected exactly 2 wheels in $DIST, found ${#wheels[@]}"
+for w in "${wheels[@]}"; do
+ stamp="$(grep -oE '[+.]g[0-9a-f]{7,}' <<<"${w##*/}" | head -1 | sed 's/^[+.]g//' || true)"
+ [ -n "$stamp" ] || die "$(basename "$w") carries no build stamp"
+ same_commit "$stamp" "$source_commit" || die "$(basename "$w") was built from $stamp, expected $source_commit"
+done
+
+say "fetched into $DIST:"
+ls -l "$DIST"
diff --git a/scripts/ci/manylinux-build.sh b/scripts/ci/manylinux-build.sh
index 4d8ff913..d2dd3ca2 100755
--- a/scripts/ci/manylinux-build.sh
+++ b/scripts/ci/manylinux-build.sh
@@ -19,8 +19,8 @@
# (default: cp312 -- the nightly/Desktop channel is cp312-only;
# the release lane passes "cp310 cp311 cp312 cp313")
# FT_MANYLINUX_RETAG retag runtime wheels linux_x86_64 -> detected manylinux (default: 0).
-# Release/PyPI lane only: shipped Desktops resolve the nightly
-# release's assets by name and expect linux_x86_64.
+# Release/PyPI lane only: shipped Desktops resolve the beta
+# release's assets (copied from nightly) by name and expect linux_x86_64.
# FREETOKEN_BUILD_NO_STAMP / _RELEASE / _DEV_STAMP / _STRIP and
# FREETOKEN_KERNEL_CACHE_* are forwarded into the container. Other
# FREETOKEN_BUILD_* vars are NOT: _CLEAN is set by this script per matrix
diff --git a/scripts/publish-wheels.sh b/scripts/publish-wheels.sh
index 600767aa..6385c71a 100755
--- a/scripts/publish-wheels.sh
+++ b/scripts/publish-wheels.sh
@@ -1,10 +1,17 @@
#!/usr/bin/env bash
#
-# Publish built engine wheels to the FreeToken-Web rolling `beta` release.
+# Publish a built engine wheel pair (runtime + kernel-cache) to a GitHub release.
#
# Usage:
# scripts/publish-wheels.sh [dist-dir] (default: ./dist)
#
+# Two kinds of target share this script:
+# - the rolling `nightly` prerelease on this repo (FREETOKEN_RELEASE_ROLLING=1): created
+# on first use; after every publish its notes are rewritten from the manifest and its
+# tag is moved to FREETOKEN_RELEASE_COMMIT. Linux-only by design.
+# - a curated channel such as FreeToken-Web's `beta` release: must already exist; only
+# the wheels and the manifest are replaced, notes and tag are left alone.
+#
# Deletes the release's previous wheels for each platform being published, THEN
# uploads the new ones -- in that order. Shipped Desktops resolve assets with a
# first-match scan over the release's asset list, so an old and a new wheel
@@ -13,25 +20,45 @@
# with write access to the target repo.
#
# After the upload, writes `engine-<platform>.json` to the release: the pair's URLs,
-# sha256 and sizes under a fixed asset name, so a Desktop resolves the pair with one
+# sha256 and sizes under a fixed asset name, so a consumer resolves the pair with one
# static download (no api.github.com, no per-IP rate limit) and scans the asset list
-# only when the manifest is missing. One file per platform -- the linux nightly and a
-# hand-run windows publish never touch each other's manifest.
+# only when the manifest is missing. One file per platform -- publishing one platform
+# never touches another platform's manifest.
#
# Environment:
-# FREETOKEN_WEB_REPO target repo (default: FlashML-org/FreeToken-Web)
-# FREETOKEN_WEB_TAG release tag (default: beta)
+# FREETOKEN_RELEASE_REPO target repo, required (e.g. FlashML-org/FreeToken)
+# FREETOKEN_RELEASE_TAG release tag, required (e.g. nightly)
+# FREETOKEN_RELEASE_ROLLING 1 = rolling-nightly mode described above (default: 0)
+# FREETOKEN_RELEASE_COMMIT full sha the rolling tag moves to; must match the wheel stamp
+# FREETOKEN_WEB_REPO / FREETOKEN_WEB_TAG older names for REPO / TAG, still honored
set -euo pipefail
-REPO="${FREETOKEN_WEB_REPO:-FlashML-org/FreeToken-Web}"
-TAG="${FREETOKEN_WEB_TAG:-beta}"
+REPO="${FREETOKEN_RELEASE_REPO:-${FREETOKEN_WEB_REPO:-}}"
+TAG="${FREETOKEN_RELEASE_TAG:-${FREETOKEN_WEB_TAG:-}}"
+ROLLING="${FREETOKEN_RELEASE_ROLLING:-0}"
+COMMIT="${FREETOKEN_RELEASE_COMMIT:-}"
DIST="${1:-dist}"
+ROLLING_TITLE="Nightly (rolling)"
say() { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
die() { printf '\033[1;31m[error]\033[0m %s\n' "$*" >&2; exit 1; }
command -v gh >/dev/null 2>&1 || die "gh not found"
+command -v jq >/dev/null 2>&1 || die "jq not found"
[ -d "$DIST" ] || die "no such dist dir: $DIST"
+{ [ -n "$REPO" ] && [ -n "$TAG" ]; } \
+ || die "set FREETOKEN_RELEASE_REPO and FREETOKEN_RELEASE_TAG -- there is no default target"
+
+case " 1 true yes on " in
+ *" $(printf '%s' "$ROLLING" | tr '[:upper:]' '[:lower:]') "*) ROLLING=1 ;;
+ *) ROLLING=0 ;;
+esac
+if [ "$ROLLING" = 1 ]; then
+ [[ "$COMMIT" =~ ^[0-9a-f]{40}$ ]] \
+ || die "FREETOKEN_RELEASE_ROLLING needs FREETOKEN_RELEASE_COMMIT set to the full 40-hex sha the tag moves to"
+ # Rolling mode force-moves its tag; a release tag must never be reachable that way.
+ [[ "$TAG" != v[0-9]* ]] || die "refusing to run in rolling mode against release tag '$TAG'"
+fi
shopt -s nullglob
wheels=("$DIST"/freetoken-*.whl "$DIST"/freetoken_kernel_cache-*.whl)
@@ -47,6 +74,9 @@ platforms="$(for w in "${wheels[@]}"; do
esac
done | sort -u)"
grep -qx UNKNOWN <<<"$platforms" && die "cannot infer the platform of every wheel in $DIST"
+if [ "$ROLLING" = 1 ] && [ "$platforms" != linux_x86_64 ]; then
+ die "the rolling release takes the linux_x86_64 pair only (got: $(tr '\n' ' ' <<<"$platforms"))"
+fi
# Publish gate: prune-then-upload replaces a platform's wheel pair WHOLESALE, so demand
# a complete, self-consistent pair up front -- exactly one runtime + one kernel-cache
@@ -54,7 +84,7 @@ grep -qx UNKNOWN <<<"$platforms" && die "cannot infer the platform of every whee
# that died between the two wheels) or a mixed-build pair must fail HERE, before any
# asset is deleted, not leave the release missing an asset it will never get back.
while IFS= read -r p; do
- rt_n=0; kc_n=0; rt_stamp=""; kc_stamp=""
+ rt_n=0; kc_n=0; rt_stamp=""; kc_stamp=""; kc_b=""
for w in "${wheels[@]}"; do
b="${w##*/}"
case "$b" in
@@ -63,7 +93,7 @@ while IFS= read -r p; do
rt_stamp="$(grep -oE '\+g[0-9a-f]{7,}' <<<"$b" | head -1 || true)"
;;
freetoken_kernel_cache-*"$p"*.whl)
- kc_n=$((kc_n + 1))
+ kc_n=$((kc_n + 1)); kc_b="$b"
kc_stamp="$(grep -oE '\.g[0-9a-f]{7,}' <<<"$b" | head -1 || true)"
;;
esac
@@ -73,13 +103,35 @@ while IFS= read -r p; do
if [ -n "$rt_stamp" ] && [ -n "$kc_stamp" ] && [ "${rt_stamp#+}" != "${kc_stamp#.}" ]; then
die "stamp mismatch for $p: runtime has ${rt_stamp}, kernel-cache has ${kc_stamp} -- these wheels are from different builds"
fi
+ if [ "$ROLLING" = 1 ]; then
+ [ -n "$rt_stamp" ] || die "the rolling release needs stamped wheels (+g<sha>); $p runtime wheel has no stamp"
+ [[ "$COMMIT" == "${rt_stamp#+g}"* ]] \
+ || die "FREETOKEN_RELEASE_COMMIT ${COMMIT:0:9} is not the commit the $p wheels were built from (${rt_stamp#+g})"
+ grep -qE '\+cu[0-9]+' <<<"$kc_b" \
+ || die "the rolling release needs a kernel-cache wheel with a +cuNNN version segment ($kc_b)"
+ fi
done <<<"$platforms"
say "publishing to $REPO tag '$TAG':"
for w in "${wheels[@]}"; do say " $(basename "$w")"; done
+# Only the rolling release is created on demand; a curated channel that is missing
+# is an operator error, not something to paper over with an empty release.
+err="$(mktemp)"
+if ! release_json="$(gh api "repos/$REPO/releases/tags/$TAG" 2>"$err")"; then
+ grep -q 'HTTP 404' "$err" || { cat "$err" >&2; die "cannot read release '$TAG' on $REPO"; }
+ [ "$ROLLING" = 1 ] || die "release '$TAG' does not exist on $REPO"
+ say "creating the rolling release '$TAG' at ${COMMIT:0:9}"
+ release_json="$(gh api -X POST "repos/$REPO/releases" \
+ -f tag_name="$TAG" -f target_commitish="$COMMIT" -f name="$ROLLING_TITLE" \
+ -f body="Automated build of main. Assets are published by scripts/publish-wheels.sh." \
+ -F prerelease=true -F draft=false -f make_latest=false)"
+fi
+rm -f "$err"
+release_id="$(jq -r '.id' <<<"$release_json")"
+existing="$(jq -r '.assets[].name' <<<"$release_json")"
+
# Prune the previous generation first (delete-then-upload; see header).
-existing="$(gh api "repos/$REPO/releases/tags/$TAG" --jq '.assets[].name')"
while IFS= read -r p; do
while IFS= read -r name; do
case "$name" in
@@ -142,6 +194,77 @@ JSON
gh release upload "$TAG" "$manifest" -R "$REPO" --clobber
done <<<"$platforms"
+mb() { awk -v b="$1" 'BEGIN { printf "%.1f MB", b / 1000000 }'; }
+render_notes() {
+ local m="$1" py cuda
+ py="$(jq -r .python "$m" | sed -E 's/^cp([0-9])([0-9]+)$/\1.\2/')"
+ cuda="$(jq -r .cuda "$m" | sed -E 's/^cu([0-9]+)([0-9])$/\1.\2/')"
+ local commit published rt_name rt_url rt_sha rt_size kc_name kc_url kc_sha kc_size
+ commit="$(jq -r .commit "$m")"
+ published="$(jq -r .published "$m")"
+ rt_name="$(jq -r .runtime.name "$m")"; rt_url="$(jq -r .runtime.url "$m")"
+ rt_sha="$(jq -r .runtime.sha256 "$m")"; rt_size="$(jq -r .runtime.size "$m")"
+ kc_name="$(jq -r .kernel_cache.name "$m")"; kc_url="$(jq -r .kernel_cache.url "$m")"
+ kc_sha="$(jq -r .kernel_cache.sha256 "$m")"; kc_size="$(jq -r .kernel_cache.size "$m")"
+ cat <<MD
+## FreeToken nightly (rolling)
+
+Automated build of \`main\`. This release is **overwritten every night**: the \`$TAG\` tag moves to the built commit and the previous wheels are deleted. Do not pin to this tag; pin to a wheel URL below, whose filename embeds the commit it was built from.
+
+**Install** (Linux x86_64, CPython $py, CUDA $cuda):
+
+\`\`\`bash
+uv pip install \\
+ "freetoken[accel] @ $rt_url" \\
+ "$kc_url"
+\`\`\`
+
+| commit | built (UTC) | python | cuda | runtime | kernel-cache |
+|---|---|---|---|---|---|
+| [\`$commit\`](https://github.com/$REPO/commit/$commit) | ${published:0:10} ${published:11:5} | cp${py/./} | cu${cuda/./} | $(mb "$rt_size") | $(mb "$kc_size") |
+
+<details><summary>sha256</summary>
+
+\`\`\`
+$rt_sha $rt_name
+$kc_sha $kc_name
+\`\`\`
+</details>
+
+Machine-readable: \`engine-linux_x86_64.json\` next to the wheels (schema 1).
+Stable releases: [PyPI \`freetoken\`](https://pypi.org/project/freetoken/) and the \`v*\` releases here.
+MD
+}
+
+# A transient failure past this point would leave the new wheels under the old notes
+# and tag, and nothing re-runs the publish; retry before giving up.
+gh_retry() {
+ local attempt
+ for attempt in 1 2 3; do
+ "$@" && return 0
+ [ "$attempt" = 3 ] || sleep $((attempt * 5))
+ done
+ return 1
+}
+
+if [ "$ROLLING" = 1 ]; then
+ notes="$manifest_dir/notes.md"
+ render_notes "$manifest_dir/engine-linux_x86_64.json" >"$notes"
+ say "rewriting release notes"
+ gh_retry gh api -X PATCH "repos/$REPO/releases/$release_id" \
+ -f name="$ROLLING_TITLE" -F prerelease=true -f make_latest=false -F body=@"$notes" >/dev/null \
+ || die "cannot rewrite the notes of release '$TAG'"
+ # Last, so the tag only ever points at a fully published build.
+ say "moving tag '$TAG' to ${COMMIT:0:9}"
+ tag_err="$manifest_dir/tag.err"
+ if ! gh_retry gh api -X PATCH "repos/$REPO/git/refs/tags/$TAG" -f sha="$COMMIT" -F force=true >/dev/null 2>"$tag_err"; then
+ # Only a tag deleted by hand while the release survived is recreated.
+ grep -qiE 'does not exist|Not Found' "$tag_err" || { cat "$tag_err" >&2; die "cannot move tag '$TAG'"; }
+ gh api -X POST "repos/$REPO/git/refs" -f ref="refs/tags/$TAG" -f sha="$COMMIT" >/dev/null \
+ || die "cannot create tag '$TAG'"
+ fi
+fi
+
say "release now carries:"
gh api "repos/$REPO/releases/tags/$TAG" \
--jq '.assets[] | select(.name | endswith(".whl")) | " \(.name) \(.digest // "no-digest") \(.updated_at)"'