Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
#!/usr/bin/env bash
# Publish a benchmark image: two cross-linked indexes, one per variant.
#
# publish.sh ID [--stamp YYYYMMDD-sha7] [--dry-run] [--work DIR]
# [--arch A | --indexes-only] [--resume] stage: immutable pins
# publish.sh ID --stamp S --promote move <ver>, <ver>-disk
#
# Staging pushes only immutable pins and writes lib/images/bench/ID/lock.json.
# --promote moves the floating tags to a staged stamp, and only when doctor
# evidence (scripts/bench-images/smoke.py, `cua-spacesd doctor --strict`) has
# passed for that stamp's exact index digests: every arch, both variants
# (CUA_BENCH_EVIDENCE, default libs/images/bench/ID/evidence).
#
# --resume finishes an interrupted run with the same --stamp: pins that
# already exist are accepted only when they hold exactly the content this
# run pushes (check-tag-safety.sh --digest), never re-pointed.
# --arch A pushes only that arch's two children (a CI job per runner arch);
# --indexes-only reads every arch's children back from their immutable tags
# and pushes the indexes and floating tags (the final CI job). Without either,
# one machine does it all.
#
# Inputs (from build-qcow2.sh / build-docker.sh), per arch in bench.json:
# rootfs docker image cua-e2e-local/bench-ID:docker-local-<arch>, or an OCI
# layout at WORK/ID/out/<arch>/rootfs-layout (qcow2 sources)
# disk WORK/ID/out/<arch>/disk.img
# Pushes, in this order (every push is checked by
# scripts/images/check-tag-safety.sh first; only ghcr.io/trycua/bench-* repos
# are accepted):
# <repo>:<ver>-<stamp>-<arch> rootfs child (immutable)
# <repo>:<ver>-disk-<stamp>-<arch> containerDisk child (immutable)
# <repo>:<ver>-disk-<stamp> containerDisk index (immutable; KubeVirt/QEMU pull this),
# ai.cua.image.variants {"rootfs": "<repo>:<ver>-<stamp>"}
# <repo>:<ver>-<stamp> rootfs index (immutable; docker/gVisor),
# ai.cua.image.variants {"containerdisk": "<repo>@<disk index>"}
# <repo>:<ver>-disk, <repo>:<ver> floating version tags (bench repos only; --promote)
# One index per variant: docker and containerd take the first child matching
# the platform and ignore annotations, and ctr pulls every matching child, so
# a single multi-variant index over-pulls.
# and writes libs/images/bench/ID/lock.json with every digest.
#
# A new ghcr.io/trycua package is private. "public" benchmarks must be made
# public once by an org admin in the package settings (GitHub has no API for
# package visibility); "private" ones stay private, and are never linked to
# the public repository.
# Credentials: CUA_BENCH_REGISTRY_CONFIG, a docker config directory used by
# crane, oras and the tag-safety check only (so the docker CLI keeps its own
# config and context), else DOCKER_CONFIG / ~/.docker. Never printed.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/../.." && pwd)"
CHECK="$REPO_ROOT/scripts/images/check-tag-safety.sh"
ID="${1:?benchmark id}"; shift
STAMP="$(date -u +%Y%m%d)-$(git -C "$HERE" rev-parse --short=7 HEAD)"
DRY=0 PROMOTE=0 ONLY_ARCH="" INDEXES_ONLY=0 RESUME=0
WORK_ROOT="${CUA_BENCH_WORK:-$HOME/.cache/cua-bench-images}"
while [ $# -gt 0 ]; do
case "$1" in
--stamp) STAMP="$2"; shift 2 ;;
--dry-run) DRY=1; shift ;;
--promote) PROMOTE=1; shift ;;
--work) WORK_ROOT="$2"; shift 2 ;;
--arch) ONLY_ARCH="$2"; shift 2 ;;
--indexes-only) INDEXES_ONLY=1; shift ;;
--resume) RESUME=1; shift ;;
*) echo "unknown option $1" >&2; exit 2 ;;
esac
done
[[ "$STAMP" =~ ^[0-9]{8}-[0-9a-f]{7}$ ]] || { echo "bad stamp $STAMP" >&2; exit 2; }
for t in crane jq docker; do command -v "$t" >/dev/null || { echo "missing $t" >&2; exit 2; }; done
bench() { python3 "$HERE/bench.py" "$@"; }
log() { echo "[publish $ID $(date +%T)] $*" >&2; }
REG_CFG="${CUA_BENCH_REGISTRY_CONFIG:-${DOCKER_CONFIG:-}}"
ORAS_CFG=()
[ -n "$REG_CFG" ] && ORAS_CFG=(--registry-config "$REG_CFG/config.json")
if [ -n "${CUA_BENCH_REGISTRY_CONFIG:-}" ]; then
crane() { DOCKER_CONFIG="$CUA_BENCH_REGISTRY_CONFIG" command crane "$@"; }
check_tags() { DOCKER_CONFIG="$CUA_BENCH_REGISTRY_CONFIG" "$CHECK" "$@"; }
else
check_tags() { "$CHECK" "$@"; }
fi
REPO="$(bench get "$ID" repository)"
VER="$(bench get "$ID" version)"
RED="$(bench get "$ID" redistribution)"
case "$REPO" in ghcr.io/trycua/bench-*) ;; *) echo "refusing $REPO: bench images go to ghcr.io/trycua/bench-* only" >&2; exit 2 ;; esac
case "$RED" in public|private-only) ;; *) echo "refusing: redistribution=$RED is never pushed (build on the user's side)" >&2; exit 2 ;; esac
ALL_ARCHES="$(bench get "$ID" arch | jq -r '.[]')"
VIS="$(bench get "$ID" visibility)"
ARCHES="$ALL_ARCHES"
if [ -n "$ONLY_ARCH" ]; then
grep -qx "$ONLY_ARCH" <<<"$ALL_ARCHES" || { echo "$ID has no arch $ONLY_ARCH" >&2; exit 2; }
ARCHES="$ONLY_ARCH"
fi
[ "$INDEXES_ONLY" = 1 ] && ARCHES=""
WORK="$WORK_ROOT/$ID/publish-$STAMP"
mkdir -p "$WORK"
if [ "$PROMOTE" = 1 ]; then
# Floating tags follow doctor evidence for the exact staged digests.
LOCK="$REPO_ROOT/libs/images/bench/$ID/lock.json"
[ "$(jq -r .stamp "$LOCK" 2>/dev/null)" = "$STAMP" ] || { echo "lock.json is not stamp $STAMP (stage it first)" >&2; exit 2; }
index="$(jq -r .index.digest "$LOCK")" disk_index="$(jq -r .disk_index.digest "$LOCK")"
EVID="${CUA_BENCH_EVIDENCE:-$REPO_ROOT/libs/images/bench/$ID/evidence}"
missing=()
for a in $ALL_ARCHES; do
for pair in "rootfs=$REPO@$index" "containerdisk=$REPO@$disk_index"; do
v="${pair%%=*}" want="${pair#*=}"
ok="$(jq -s --arg v "$v" --arg a "$a" --arg want "$want" \
'[.[] | select(.variant == $v and .arch == $a and .pinned_ref == $want and .ok == true and .doctor == "pass")] | length' \
"$EVID"/*.json 2>/dev/null || echo 0)"
[ "${ok:-0}" -gt 0 ] || missing+=("$v/$a ($want)")
done
done
if [ ${#missing[@]} -gt 0 ]; then
echo "REFUSED: no passing doctor evidence in $EVID for:" >&2; printf ' %s\n' "${missing[@]}" >&2; exit 1
fi
[ "$DRY" = 1 ] && { log "evidence complete; dry run, not promoting"; exit 0; }
check_tags --moving "$REPO:$VER-disk" "$REPO:$VER"
crane tag "$REPO@$disk_index" "$VER-disk"
crane tag "$REPO@$index" "$VER"
[ "$(crane digest "$REPO:$VER")" = "$index" ] && [ "$(crane digest "$REPO:$VER-disk")" = "$disk_index" ] \
|| { echo "floating tags do not point at the staged indexes" >&2; exit 1; }
jq '.promoted = true' "$LOCK" >"$LOCK.tmp" && mv "$LOCK.tmp" "$LOCK"
log "promoted $REPO:$VER -> $index, $REPO:$VER-disk -> $disk_index"
exit 0
fi
# Every ref this run creates must be new (or already hold the same content).
pins=()
[ -z "$ONLY_ARCH" ] && pins+=("$REPO:$VER-$STAMP" "$REPO:$VER-disk-$STAMP")
for a in $ARCHES; do pins+=("$REPO:$VER-$STAMP-$a" "$REPO:$VER-disk-$STAMP-$a"); done
[ ${#pins[@]} -eq 0 ] || [ "$RESUME" = 1 ] || check_tags "${pins[@]}"
# ghcr has no API to set a package's visibility. Before a private
# benchmark's first large push,
# a tiny probe image (pushed by digest, untagged) creates the package and its
# visibility is read back; anything but "private" aborts before real content
# is pushed.
package_visibility() {
# gh prints the error body to stdout on a 404, so only trust a success.
local out
if out="$(gh api "/orgs/trycua/packages/container/${REPO#ghcr.io/trycua/}" --jq .visibility 2>/dev/null)"; then
echo "$out"
else
echo unknown
fi
}
if [ "$VIS" = private ] && [ "$DRY" = 0 ]; then
command -v gh >/dev/null || { echo "gh is needed to verify the private package's visibility" >&2; exit 2; }
cur="$(package_visibility)"
if [ "$cur" = unknown ]; then
log "creating the private package with a probe image"
printf '{"config":{"Labels":{"ai.cua.image.probe":"visibility"}}}' >"$WORK/probe.json"
bench layout "$WORK/probe" --platform linux/amd64 --config "$WORK/probe.json" >/dev/null
crane push "$WORK/probe" "$REPO@$(jq -r '.manifests[0].digest' "$WORK/probe/index.json")" >&2
for _ in 1 2 3 4 5 6; do cur="$(package_visibility)"; [ "$cur" != unknown ] && break; sleep 5; done
fi
[ "$cur" = private ] || { echo "REFUSED: $REPO is '$cur', not private; fix it in the org's package settings" >&2; exit 1; }
log "package visibility: private"
fi
# push_layout DIR REF -> digest (crane pushes the single manifest of an OCI layout)
push_layout() {
set -e
local dir="$1" ref="$2" want
want="$(jq -r '.manifests[0].digest' "$dir/index.json")"
if [ "$DRY" = 1 ]; then echo "$want"; return; fi
check_tags --digest "$want" "$ref" >&2
crane push "$dir" "$ref" >&2
local got; got="$(crane digest "$ref")"
[ "$got" = "$want" ] || { echo "pushed $ref but it is $got, not $want" >&2; exit 1; }
echo "$want"
}
# push_index FILE REF -> digest
push_index() {
set -e
local file="$1" ref="$2" want
want="sha256:$(shasum -a 256 "$file" | cut -d' ' -f1)"
if [ "$DRY" = 1 ]; then echo "$want"; return; fi
check_tags --digest "$want" "$ref" >&2
crane manifest "$ref" >/dev/null 2>&1 && [ "$(crane digest "$ref")" = "$want" ] && { echo "$want"; return; }
oras manifest push ${ORAS_CFG[@]+"${ORAS_CFG[@]}"} --media-type application/vnd.oci.image.index.v1+json "$ref" "$file" >&2
local got; got="$(crane digest "$ref")"
[ "$got" = "$want" ] || { echo "pushed $ref but it is $got, not $want" >&2; exit 1; }
echo "$want"
}
desc() { # DIR PLATFORM -> descriptor json file
jq --arg p "$2" '.manifests[0] + {platform: {architecture: ($p|split("/")[1]), os: "linux"}}' "$1/index.json" >"$1.desc.json"
echo "$1.desc.json"
}
rootfs_descs=() disk_descs=() record_children="[]"
for a in $ARCHES; do
src="cua-e2e-local/bench-$ID:docker-local-$a"
layout="$WORK_ROOT/$ID/out/$a/rootfs-layout"
disk="$WORK_ROOT/$ID/out/$a/disk.img"
[ -f "$disk" ] || { echo "missing $disk (build it first)" >&2; exit 1; }
rm -rf "$WORK/rootfs-$a" "$WORK/rootfs-$a.tar"
if [ -f "$layout/index.json" ]; then
log "rootfs $a: OCI layout $layout"
cp -R "$layout" "$WORK/rootfs-$a"
else
docker image inspect "$src" >/dev/null || { echo "missing $src or $layout (build it first)" >&2; exit 1; }
log "rootfs $a: docker save $src"
docker save --platform "linux/$a" -o "$WORK/rootfs-$a.tar" "$src"
label_args=(); while IFS= read -r l; do label_args+=(--label "$l"); done < <(bench labels "$ID" --variant rootfs --stamp "$STAMP" | jq -r 'to_entries[] | "\(.key)=\(.value)"')
drop=(); [ "$VIS" = private ] && drop=(--drop-label org.opencontainers.image.source)
bench docker-layout "$WORK/rootfs-$a.tar" "$WORK/rootfs-$a" --platform "linux/$a" "${label_args[@]}" ${drop[@]+"${drop[@]}"} >/dev/null
rm -f "$WORK/rootfs-$a.tar"
fi
rd="$(push_layout "$WORK/rootfs-$a" "$REPO:$VER-$STAMP-$a")"
log " $REPO:$VER-$STAMP-$a -> $rd"
rootfs_descs+=(--rootfs "$(desc "$WORK/rootfs-$a" "linux/$a")")
log "containerdisk $a: $(du -h "$disk" | cut -f1) $disk"
rm -rf "$WORK/disk-$a"; mkdir -p "$WORK/disk-$a.layer"
bench disk-layer "$disk" "$WORK/disk-$a.layer/disk.tar" >/dev/null
jq -n --argjson l "$(bench labels "$ID" --variant containerdisk --stamp "$STAMP")" '{config: {Labels: $l}}' >"$WORK/disk-$a.config.json"
bench layout "$WORK/disk-$a" --platform "linux/$a" --config "$WORK/disk-$a.config.json" \
--layer "$WORK/disk-$a.layer/disk.tar.json" >/dev/null
dd="$(push_layout "$WORK/disk-$a" "$REPO:$VER-disk-$STAMP-$a")"
log " $REPO:$VER-disk-$STAMP-$a -> $dd"
rm -rf "$WORK/disk-$a.layer" "$WORK/disk-$a/blobs"
disk_descs+=(--disk "$(desc "$WORK/disk-$a" "linux/$a")")
record_children="$(jq -c --arg a "$a" --arg r "$rd" --arg d "$dd" '. + [{arch: $a, rootfs: $r, containerdisk: $d}]' <<<"$record_children")"
done
if [ -n "$ONLY_ARCH" ]; then
jq -n --arg stamp "$STAMP" --argjson children "$record_children" '{stamp: $stamp, children: $children}' >"$WORK/record-$ONLY_ARCH.json"
cat "$WORK/record-$ONLY_ARCH.json"
exit 0
fi
if [ "$INDEXES_ONLY" = 1 ]; then
# Children pushed by earlier --arch runs, read back from their pins.
for a in $ALL_ARCHES; do
for kind in rootfs disk; do
tag="$VER-$STAMP-$a"; [ "$kind" = disk ] && tag="$VER-disk-$STAMP-$a"
m="$(crane manifest "$REPO:$tag")"
jq -n --arg d "$(crane digest "$REPO:$tag")" --argjson size "$(crane manifest "$REPO:$tag" | wc -c | tr -d ' ')" \
--arg mt "$(jq -r .mediaType <<<"$m")" --arg a "$a" \
'{mediaType: $mt, digest: $d, size: $size, platform: {architecture: $a, os: "linux"}}' >"$WORK/$kind-$a.desc.json"
if [ "$kind" = rootfs ]; then rootfs_descs+=(--rootfs "$WORK/$kind-$a.desc.json"); else disk_descs+=(--disk "$WORK/$kind-$a.desc.json"); fi
done
record_children="$(jq -c --arg a "$a" --arg r "$(jq -r .digest "$WORK/rootfs-$a.desc.json")" --arg d "$(jq -r .digest "$WORK/disk-$a.desc.json")" '. + [{arch: $a, rootfs: $r, containerdisk: $d}]' <<<"$record_children")"
done
fi
log "indexes"
bench index "$ID" --stamp "$STAMP" "${rootfs_descs[@]}" "${disk_descs[@]}" "$WORK/idx" >/dev/null
disk_index="$(push_index "$WORK/idx/disk-index.json" "$REPO:$VER-disk-$STAMP")"
log " $REPO:$VER-disk-$STAMP -> $disk_index"
bench index "$ID" --stamp "$STAMP" "${rootfs_descs[@]}" "${disk_descs[@]}" --disk-digest "$disk_index" "$WORK/idx" >/dev/null
index="$(push_index "$WORK/idx/index.json" "$REPO:$VER-$STAMP")"
log " $REPO:$VER-$STAMP -> $index"
log "staged; promote with: $0 $ID --stamp $STAMP --promote (after the doctor smokes pass)"
jq -n --arg stamp "$STAMP" --arg index "$index" --arg disk "$disk_index" --argjson children "$record_children" \
--arg repo "$REPO" --arg ver "$VER" --argjson dry "$DRY" '{
stamp: $stamp, dry_run: ($dry == 1),
promoted: false,
index: {ref: "\($repo):\($ver)-\($stamp)", digest: $index, floating: "\($repo):\($ver)"},
disk_index: {ref: "\($repo):\($ver)-disk-\($stamp)", digest: $disk, floating: "\($repo):\($ver)-disk"},
children: $children,
pinned: "\($repo)@\($index)"}' >"$WORK/record.json"
if [ "$DRY" = 0 ]; then
bench lock "$ID" --from "$WORK/record.json"
command -v gh >/dev/null && log "package visibility now: $(package_visibility) (bench.json: $VIS)"
fi
cat "$WORK/record.json"