Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(cloud): substitute local stack binaries from the S3 nix cache (#5804)
teo8810bd792f6c
diff --git a/.cursor/cloud-lib.sh b/.cursor/cloud-lib.shindex 81f4b8d3b5..b455e01260 100755--- a/.cursor/cloud-lib.sh+++ b/.cursor/cloud-lib.sh@@ -1,12 +1,7 @@-# Shared by install.sh / start.sh / stack.sh. Sourced, not executed.-# Persistent caches live under $HOME so they survive `git checkout` into /workspace.-WORKSPACE_ROOT='/workspace'CACHE_ROOT="${HOME}/.cache/macro-cloud"TARGET_CACHE="${CACHE_ROOT}/target"-FRONTEND_CACHE="${CACHE_ROOT}/frontend"export MACRO_STACK_SNAPSHOT_DIR="${CACHE_ROOT}/stack-snapshots"-STACK_SNAPSHOT_IMAGE='macro-cloud-stack-snapshot:prepared'LOG_DIR="${HOME}/.cursor-cloud"MACRODB_URL='postgres://user:password@localhost:5432/macrodb'@@ -15,35 +10,93 @@ DOCKER_IPTABLES_BACKEND='/usr/sbin/iptables-legacy'DOCKER_IP6TABLES_BACKEND='/usr/sbin/ip6tables-legacy'NIX_BIN='/nix/var/nix/profiles/default/bin/nix'NIX_SOCK='/nix/var/nix/daemon-socket/socket'+NIX_CACHE_URL='s3://macro-nix-cache?region=us-east-1&compression=zstd'+NIX_CACHE_PUBLIC_KEY='nix-cache.macro.com-1:UtlRPa6ac+o4IfY+wV8KUS+X0XPU0YMv18lPWEDYN5k='+LOCAL_STACK_BINS="${CACHE_ROOT}/local-stack-bins"export DATABASE_URL="${MACRODB_URL}"-mkdir -p "${LOG_DIR}" "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"+mkdir -p "${LOG_DIR}" "${TARGET_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"++nix_base_conf() {+ echo 'experimental-features = nix-command flakes'+ echo "trusted-users = root $(id -un)"+ echo 'build-users-group = nixbld'+ echo 'sandbox = false'+}ensure_nix_installed() {if [ -x "${NIX_BIN}" ]; thenreturn 0fiecho "cursor-cloud: installing Determinate Nix"+ local conf_args=()+ local line+ while IFS= read -r line; do+ conf_args+=(--extra-conf "${line}")+ done < <(nix_base_conf)curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \https://install.determinate.systems/nix |sudo sh -s -- install linux \--init none \--no-confirm \- --extra-conf "trusted-users = root $(id -un)" \- --extra-conf "sandbox = false"+ "${conf_args[@]}"test -x "${NIX_BIN}"}ensure_nix_daemon() {ensure_nix_installed++ local cache_enabled=false+ if [ -n "${NIX_CACHE_AWS_ACCESS_KEY_ID:-}" ] \+ && [ -n "${NIX_CACHE_AWS_SECRET_ACCESS_KEY:-}" ]; then+ cache_enabled=true+ fi++ local desired_conf+ desired_conf="$(mktemp)"+ {+ nix_base_conf+ if "${cache_enabled}"; then+ echo "extra-substituters = ${NIX_CACHE_URL}"+ echo "extra-trusted-public-keys = ${NIX_CACHE_PUBLIC_KEY}"+ echo 'narinfo-cache-negative-ttl = 30'+ fi+ } >"${desired_conf}"++ local config_changed=false+ sudo mkdir -p /etc/nix+ if ! sudo cmp -s "${desired_conf}" /etc/nix/nix.conf; then+ sudo install -m 0644 "${desired_conf}" /etc/nix/nix.conf+ sudo pkill -x nix-daemon >/dev/null 2>&1 || true+ config_changed=true+ fi+ rm -f "${desired_conf}"++ if "${config_changed}"; then+ local config_wait=0+ while sudo pgrep -x nix-daemon >/dev/null 2>&1 && [ "${config_wait}" -lt 30 ]; do+ config_wait=$((config_wait + 1))+ sleep 1+ done+ fi+if "${NIX_BIN}" ping-store >/dev/null 2>&1; thenreturn 0fi- # Disk snapshots preserve socket path entries, not the process listening on- # them. Remove the stale entry only after a real store probe has failed.++ # Disk snapshots preserve dead socket entries.sudo rm -f "${NIX_SOCK}": >"${LOG_DIR}/nix-daemon.log"- sudo setsid "${NIX_BIN}" daemon >>"${LOG_DIR}/nix-daemon.log" 2>&1 </dev/null &+ local daemon_env=()+ if "${cache_enabled}"; then+ # The daemon performs substitution, so it owns the S3 credentials.+ daemon_env+=(+ "AWS_ACCESS_KEY_ID=${NIX_CACHE_AWS_ACCESS_KEY_ID}"+ "AWS_SECRET_ACCESS_KEY=${NIX_CACHE_AWS_SECRET_ACCESS_KEY}"+ )+ fi+ sudo setsid env "${daemon_env[@]}" "${NIX_BIN}" daemon \+ >>"${LOG_DIR}/nix-daemon.log" 2>&1 </dev/null &local n=0while [ "${n}" -lt 30 ]; doif "${NIX_BIN}" ping-store >/dev/null 2>&1; then@@ -58,6 +111,12 @@ ensure_nix_daemon() {}ensure_docker_iptables_backend() {+ if [ ! -x /usr/bin/update-alternatives ] \+ || [ ! -x "${DOCKER_IPTABLES_BACKEND}" ] \+ || [ ! -x "${DOCKER_IP6TABLES_BACKEND}" ]; then+ return 0+ fi+local desired_backenddesired_backend="$(readlink -f "${DOCKER_IPTABLES_BACKEND}")"local desired_ip6_backend@@ -65,12 +124,7 @@ ensure_docker_iptables_backend() {local current_backendcurrent_backend="$(readlink -f /etc/alternatives/iptables 2>/dev/null || true)"- test -x "${DOCKER_IPTABLES_BACKEND}"- test -x "${DOCKER_IP6TABLES_BACKEND}"-- # The cloud image can preserve a legacy FORWARD DROP policy while apt selects- # iptables-nft. Make Docker program the enforcing legacy table instead of- # opening FORWARD or bypassing Docker's per-network isolation chains.+ # Reconcile the backend with a preserved legacy FORWARD policy.if [ "${current_backend}" != "${desired_backend}" ]; thensudo /usr/bin/update-alternatives \--set iptables "${DOCKER_IPTABLES_BACKEND}"@@ -80,19 +134,25 @@ ensure_docker_iptables_backend() {sudo /usr/bin/update-alternatives \--set ip6tables "${DOCKER_IP6TABLES_BACKEND}"fi-- current_backend="$(readlink -f /etc/alternatives/iptables)"- local current_ip6_backend- current_ip6_backend="$(readlink -f /etc/alternatives/ip6tables)"- test "${current_backend}" = "${desired_backend}"- test "${current_ip6_backend}" = "${desired_ip6_backend}"}ensure_dockerd() {ensure_docker_iptables_backend++ local dockerd_bin+ if [ -x /usr/bin/dockerd ]; then+ dockerd_bin='/usr/bin/dockerd'+ else+ dockerd_bin="$(command -v dockerd || true)"+ fi+ if [ -z "${dockerd_bin}" ] || [ ! -x "${dockerd_bin}" ]; then+ echo "dockerd not found in /usr/bin or PATH" >&2+ return 1+ fi+local storage_driver='vfs'local storage_args=()- if [ -x /usr/bin/fuse-overlayfs ]; then+ if command -v fuse-overlayfs >/dev/null 2>&1; thenstorage_driver='fuse-overlayfs'fiif ! sudo /usr/bin/grep -q '"storage-driver"[[:space:]]*:' /etc/docker/daemon.json 2>/dev/null; then@@ -104,11 +164,10 @@ ensure_dockerd() {return 0fifi- # As with Nix, a snapshot can contain a dead Unix socket. Docker does not- # reliably replace it while starting, so clear it after the API probe fails.+ # Disk snapshots preserve dead socket entries.sudo rm -f "${DOCKER_SOCK}": >"${LOG_DIR}/dockerd.log"- sudo setsid /usr/bin/dockerd "${storage_args[@]}" \+ sudo setsid env "PATH=${PATH}" "${dockerd_bin}" "${storage_args[@]}" \>>"${LOG_DIR}/dockerd.log" 2>&1 </dev/null &local n=0while [ "${n}" -lt 60 ]; do@@ -162,83 +221,25 @@ remove_workspace_target_mounts() {done}-# Move /workspace/target into $HOME (same filesystem: rename). Recreate the-# symlink after every checkout so Cargo's incremental cache survives it.ensure_persistent_caches() {- mkdir -p "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"+ mkdir -p "${TARGET_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"remove_workspace_target_mountsif [ -e "${WORKSPACE_ROOT}/target" ] && [ ! -L "${WORKSPACE_ROOT}/target" ]; then- sudo chown -R "$(workspace_owner):$(workspace_group)" "${WORKSPACE_ROOT}/target"- if [ -x "${WORKSPACE_ROOT}/target/x86_64-unknown-linux-gnu/debug/document_storage_service" ] \- && [ ! -x "${TARGET_CACHE}/x86_64-unknown-linux-gnu/debug/document_storage_service" ]; then- echo "cursor-cloud: moving ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"- rm -rf "${TARGET_CACHE}"- mv "${WORKSPACE_ROOT}/target" "${TARGET_CACHE}"- else- echo "cursor-cloud: replacing ${WORKSPACE_ROOT}/target with cache symlink"- rm -rf "${WORKSPACE_ROOT}/target"- fi+ sudo rm -rf "${WORKSPACE_ROOT}/target"fimkdir -p "${TARGET_CACHE}"ln -sfn "${TARGET_CACHE}" "${WORKSPACE_ROOT}/target"- sudo chown -R "$(workspace_owner):$(workspace_group)" "${TARGET_CACHE}"- echo "cursor-cloud: ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"-- if [ ! -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ] && [ -f "${FRONTEND_CACHE}/index.html" ]; then- mkdir -p "${WORKSPACE_ROOT}/apps/web/dist"- cp -a "${FRONTEND_CACHE}/." "${WORKSPACE_ROOT}/apps/web/dist/"- echo "cursor-cloud: restored frontend bundle from cache"- elif [ -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ]; then- cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"+ if [ "$(stat -c '%U' "${TARGET_CACHE}")" != "$(workspace_owner)" ]; then+ sudo chown -R "$(workspace_owner):$(workspace_group)" "${TARGET_CACHE}"fi-- if [ -d "${WORKSPACE_ROOT}/infra/local/generated/.snapshots" ]; then- cp -a "${WORKSPACE_ROOT}/infra/local/generated/.snapshots/." "${MACRO_STACK_SNAPSHOT_DIR}/"- fi-}--bake_stack_snapshot_image() {- local key="$1"- local snapshot_dir="$2"-- test -f "${snapshot_dir}/manifest.json"- docker build \- --tag "${STACK_SNAPSHOT_IMAGE}" \- --file - \- "${snapshot_dir}" <<EOF-FROM alpine:3-LABEL com.macro.stack-snapshot-key="${key}"-COPY . /snapshot/-EOF- docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null+ echo "cursor-cloud: ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"}-restore_stack_snapshot_image() {- if ! docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null 2>&1; then- echo "cursor-cloud: prepared stack snapshot image not found"- return 0- fi-- local key- key="$(docker image inspect \- --format '{{ index .Config.Labels "com.macro.stack-snapshot-key" }}' \- "${STACK_SNAPSHOT_IMAGE}")"- case "${key}" in- ''|*[!0-9a-f]*)- echo "prepared stack snapshot image has invalid key: ${key}" >&2- return 1- ;;- esac-- local destination="${MACRO_STACK_SNAPSHOT_DIR}/${key}"- mkdir -p "${destination}"- docker run --rm \- --user "$(id -u):$(id -g)" \- --volume "${destination}:/restore" \- "${STACK_SNAPSHOT_IMAGE}" \- sh -ceu 'cp -R /snapshot/. /restore/'- test -f "${destination}/manifest.json"- echo "cursor-cloud: restored stack snapshot ${key} from image"+build_local_stack_binaries() {+ (+ \cd "${WORKSPACE_ROOT}"+ nix build "${WORKSPACE_ROOT}#local-stack-binaries" --out-link "${LOCAL_STACK_BINS}"+ )}diff --git a/.cursor/infra.sh b/.cursor/infra.shnew file mode 100644index 0000000000..16115b5810--- /dev/null+++ b/.cursor/infra.sh@@ -0,0 +1,25 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++# On-demand infra: dockerd, then Postgres and Redis. Run this before+# DB-backed `cargo test`; boot (start.sh) deliberately starts nothing.++# shellcheck source=cloud-lib.sh+source "${SCRIPT_DIR}/cloud-lib.sh"++if ! in_pinned_nix_shell; then+ ensure_nix_daemon+ reenter_pinned_nix_shell "${SCRIPT_DIR}/infra.sh" "$@"+fi++ensure_dockerd+ensure_persistent_caches++\cd "${WORKSPACE_ROOT}"+just run_dbs -d++pg_isready -h 127.0.0.1 -p 5432++echo "cursor-cloud infra: ready"diff --git a/.cursor/install.sh b/.cursor/install.shindex 4e3ad3c359..3572c80082 100755--- a/.cursor/install.sh+++ b/.cursor/install.sh@@ -6,30 +6,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"# shellcheck source=cloud-lib.shsource "${SCRIPT_DIR}/cloud-lib.sh"-ensure_apt_packages() {- local pkg- local missing=()- for pkg in fuse-overlayfs libssl-dev pkg-config postgresql-client; do- if ! dpkg -s "${pkg}" >/dev/null 2>&1; then- missing+=("${pkg}")- fi- done- # Do not replace Docker CE on bases that already provide a complete engine.- if [ ! -x /usr/bin/docker ] || [ ! -x /usr/bin/dockerd ]; then- missing+=(docker.io docker-buildx docker-compose-v2)- fi- if [ "${#missing[@]}" -ne 0 ]; then- sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq- sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \- -o Dpkg::Options::=--force-confdef \- -o Dpkg::Options::=--force-confold \- "${missing[@]}"- fi- test -x /usr/bin/dockerd- /usr/bin/docker buildx version- /usr/bin/docker compose version-}-crate_env_paths() {awk '$0 == "setup_test_envs:" { p = 1; next }@@ -41,7 +17,6 @@ crate_env_paths() {' /workspace/tooling/just/rust.just}-# setup_test_envs always appends; write each DATABASE_URL at most once.ensure_test_envs() {local fwhile IFS= read -r f; do@@ -54,20 +29,8 @@ ensure_test_envs() {done < <(crate_env_paths)}-build_frontend_artifact() {- bun install --frozen-lockfile- (- cd "${WORKSPACE_ROOT}/apps/web"- MODE=development NODE_ENV=production VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin \- bun run --bun build- )- test -f "${WORKSPACE_ROOT}/apps/web/dist/index.html"- mkdir -p "${FRONTEND_CACHE}"- cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"-}-cleanup_stack() {- cd "${WORKSPACE_ROOT}"+ \cd "${WORKSPACE_ROOT}"timeout --kill-after=10s 2m just --quiet stack down}@@ -75,57 +38,15 @@ cleanup_stack_best_effort() {cleanup_stack >/dev/null 2>&1 || true}-prepare_durable_stack() {- # `infra-only` is the existing finite CI bake mode: it materializes every- # Compose image, initializes the real infra, and saves/restores the- # content-addressed volume snapshot without starting secret-dependent apps.- just stack up \- --infra-only \- --no-doppler \- --build-aux-services \- --json-- local snapshot_json- snapshot_json="$(- cargo run --quiet --manifest-path Cargo.toml -p xtask_local --features local-stack -- \- stack snapshot --json- )"- local snapshot_fields=()- mapfile -t snapshot_fields < <(python3 -c '-import json-import sys--snapshot = json.load(sys.stdin)-key = snapshot["key"]-if not snapshot["present"]:- raise SystemExit(f"stack snapshot missing for key {key}")-print(key)-print(snapshot["dir"])-' <<<"${snapshot_json}")- local snapshot_key="${snapshot_fields[0]}"- local snapshot_dir="${snapshot_fields[1]}"-- bake_stack_snapshot_image "${snapshot_key}" "${snapshot_dir}"- local image_id- image_id="$(docker image inspect --format '{{.Id}}' "${STACK_SNAPSHOT_IMAGE}")"-- echo "cursor-cloud install: stack snapshot ${snapshot_key} at ${snapshot_dir}"- echo "cursor-cloud install: stack snapshot image ${STACK_SNAPSHOT_IMAGE} (${image_id})"-}-if ! in_pinned_nix_shell; then- ensure_apt_packagesensure_nix_daemon- ensure_dockerdreenter_pinned_nix_shell "${SCRIPT_DIR}/install.sh" "$@"fi+ensure_dockerdensure_persistent_caches-docker pull pgvector/pgvector:pg18-docker pull redis/redis-stack:latest--cd "${WORKSPACE_ROOT}"+\cd "${WORKSPACE_ROOT}"just run_dbs -dtrap cleanup_stack_best_effort EXIT@@ -138,14 +59,13 @@ cargo test --no-run -p macro_db_clientecho "cursor-cloud install: test-ready"-# Build tools own cache validity: Bun, Cargo, and BuildKit compare the current-# source/lock/toolchain inputs instead of trusting surviving files or image tags.-build_frontend_artifact-echo "cursor-cloud install: frontend artifact"+bun install --frozen-lockfile+echo "cursor-cloud install: bun cache ready"++build_local_stack_binaries+echo "cursor-cloud install: local stack binaries ready"-# The bake primitive builds the runtime image and complete Rust binary inventory-# before initializing and snapshotting infra.-prepare_durable_stack+just stack up --infra-only --no-doppler --build-aux-services --jsoncleanup_stacktrap - EXITdiff --git a/.cursor/rebuild.sh b/.cursor/rebuild.shnew file mode 100755index 0000000000..2eb9b197d9--- /dev/null+++ b/.cursor/rebuild.sh@@ -0,0 +1,22 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++# Backend-edit flow, as one verb: rebuild the Nix stack binaries and recycle+# the stack onto them. This replaces `just stack update`, which cannot write+# into the read-only Nix binaries directory.++# shellcheck source=cloud-lib.sh+source "${SCRIPT_DIR}/cloud-lib.sh"++if ! in_pinned_nix_shell; then+ ensure_nix_daemon+ reenter_pinned_nix_shell "${SCRIPT_DIR}/rebuild.sh" "$@"+fi++/usr/bin/bash "${SCRIPT_DIR}/infra.sh"++\cd "${WORKSPACE_ROOT}"+just stack down+exec /usr/bin/bash "${SCRIPT_DIR}/stack.sh"diff --git a/.cursor/stack.sh b/.cursor/stack.shindex 871d4fdde0..fa6177f42a 100755--- a/.cursor/stack.sh+++ b/.cursor/stack.sh@@ -3,23 +3,21 @@ set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-# On-demand product stack. start.sh is infra-only so cargo-test agents do not-# boot FusionAuth. Cargo, Bun, and BuildKit incrementally reconcile the checked-# out branch against the durable main-build caches before services start.+# On-demand product stack. Boot (start.sh) starts nothing; infra.sh brings up+# dockerd and the databases first. After backend edits, run rebuild.sh.# shellcheck source=cloud-lib.shsource "${SCRIPT_DIR}/cloud-lib.sh"if ! in_pinned_nix_shell; thenensure_nix_daemon- ensure_dockerdreenter_pinned_nix_shell "${SCRIPT_DIR}/stack.sh" "$@"fi-/usr/bin/bash "${SCRIPT_DIR}/start.sh"+/usr/bin/bash "${SCRIPT_DIR}/infra.sh"-cd "${WORKSPACE_ROOT}"-restore_stack_snapshot_image-just stack up --no-doppler --build-aux-services+\cd "${WORKSPACE_ROOT}"+build_local_stack_binaries+just stack up --no-doppler --build-aux-services --binaries-dir "${LOCAL_STACK_BINS}/bin"echo "cursor-cloud stack: app ready"diff --git a/.cursor/start.sh b/.cursor/start.shindex 2eb37885f1..bddaf260f4 100755--- a/.cursor/start.sh+++ b/.cursor/start.sh@@ -6,17 +6,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"# shellcheck source=cloud-lib.shsource "${SCRIPT_DIR}/cloud-lib.sh"-if ! in_pinned_nix_shell; then- ensure_nix_daemon- ensure_dockerd- reenter_pinned_nix_shell "${SCRIPT_DIR}/start.sh" "$@"-fi-+# Boot path: keep this near-empty so sessions and subagents start instantly.+# On-demand work lives in infra.sh (databases) and stack.sh (product stack).+ensure_nix_daemonensure_persistent_caches-cd "${WORKSPACE_ROOT}"-just run_dbs -d--pg_isready -h 127.0.0.1 -p 5432--echo "cursor-cloud start: infra ready"+echo "cursor-cloud start: ready"diff --git a/.github/workflows/push_local_stack_binaries.yml b/.github/workflows/push_local_stack_binaries.ymlnew file mode 100644index 0000000000..9e11cb232f--- /dev/null+++ b/.github/workflows/push_local_stack_binaries.yml@@ -0,0 +1,76 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/push_local_stack_binaries.rs+name: Push local stack binaries+'on':+ push:+ branches:+ - main+ paths:+ - Cargo.toml+ - Cargo.lock+ - rust-toolchain.toml+ - crates/**+ - services/**+ - nix/**+ - flake.nix+ - flake.lock+ - '.github/actions/setup-nix/**'+ - '.github/actions/teardown-nix/**'+ - '.github/workflows/push_local_stack_binaries.yml'+ workflow_dispatch: {}+permissions:+ contents: read+jobs:+ push:+ name: Build and push local-stack-binaries+ runs-on: namespace-profile-linux-mid+ steps:+ - name: Checkout Repo+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd+ with:+ persist-credentials: 'false'+ - name: Mount /nix cache volume+ uses: namespacelabs/nscloud-cache-action@15799a6b54e5765f85b2aac25b3f0df43ed571c0+ with:+ cache: nix+ continue-on-error: true+ - name: Setup Nix+ uses: './.github/actions/setup-nix'+ with:+ nix-cache-url: ${{ vars.NIX_CACHE_URL }}+ nix-cache-public-key: ${{ vars.NIX_CACHE_PUBLIC_KEY }}+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ - name: Build local stack binaries+ run: |-+ set -euo pipefail+ nix build --print-build-logs ".#local-stack-binaries"+ echo "Local stack binaries realised into /nix/store."+ shell: bash+ - name: Push to nix cache+ run: |+ set -uo pipefail+ if [[ -z "${NIX_CACHE_URL:-}" || -z "${NIX_CACHE_SIGNING_KEY:-}" ]]; then+ echo "nix cache not configured; skipping push"+ exit 0+ fi+ key="$RUNNER_TEMP/nix-cache-signing-key"+ printf '%s\n' "$NIX_CACHE_SIGNING_KEY" > "$key"+ sep='?'; [[ "$NIX_CACHE_URL" == *\?* ]] && sep='&'+ # shellcheck disable=SC2086 # TARGETS is a list on purpose+ nix copy --to "${NIX_CACHE_URL}${sep}secret-key=${key}" $TARGETS \+ || echo "::warning::nix cache push failed (non-fatal)"+ rm -f "$key"+ shell: bash+ env:+ TARGETS: '.#local-stack-binaries'+ NIX_CACHE_URL: ${{ vars.NIX_CACHE_URL }}+ NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }}+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ - name: Teardown Nix+ if: always()+ uses: './.github/actions/teardown-nix'+concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: truediff --git a/CLAUDE.md b/CLAUDE.mdindex d6baf9a84e..5d73206932 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -317,12 +317,18 @@ don't inject it directly into the error message.## Cursor Cloud specific instructions-Environment scripts are `.cursor/install.sh` (durable files), `.cursor/start.sh` (dockerd, then Postgres and Redis), and `.cursor/stack.sh` (on-demand product stack).+`.cursor/install.sh` prepares the durable caches, databases, test dependencies, frontend dependencies, service binaries, and stack init snapshot. `.cursor/start.sh` runs at boot and starts nothing beyond the nix daemon, so sessions and subagents are usable immediately. `.cursor/infra.sh` starts Docker, Postgres, and Redis on demand. `.cursor/stack.sh` starts the on-demand product stack. `.cursor/rebuild.sh` rebuilds the stack after backend edits. These scripts are the only supported entry points; each re-enters the pinned nix shell itself, so run them with plain `bash` from any environment.-After boot, Postgres should already be up. If `:5432` is closed, run `bash .cursor/start.sh`.+Nix is the only host dependency. The pinned dev shell supplies the Docker CLI and daemon, Compose, `fuse-overlayfs`, and OpenSSH. `ssh-keygen` must come from this shell.-Cargo `target/`, the frontend bundle, and the stack init snapshot live under `$HOME/.cache/macro-cloud` and are symlinked or copied back into `/workspace`. A git checkout wipes `/workspace/target`. Cargo and BuildKit decide whether the next bake is incremental.+Service binaries come from the private S3 Nix cache. A sibling workflow on push to main (`push_local_stack_binaries.yml`) builds and pushes `.#local-stack-binaries`. It is not part of the deploy pipeline. Cursor Cloud realizes the aggregate with `nix build .#local-stack-binaries`.-Infra-ready is enough for `cargo test -p`. Product-ready is `bash .cursor/stack.sh`, which runs `just stack up --no-doppler --build-aux-services`. Install always bakes with `--infra-only --build-aux-services`. The pinned nix shell puts OpenSSH on PATH so `ssh-keygen` is not `/exec-daemon/ssh-keygen`.+Set `NIX_CACHE_AWS_ACCESS_KEY_ID` and `NIX_CACHE_AWS_SECRET_ACCESS_KEY` as Cursor environment secrets. The IAM credentials need read-only access to the cache bucket.-Do not follow the old `docker compose -f docker/docker-compose.yml up -d postgres` line. It skips `create_networks` and Redis.+Nothing runs after boot. Before DB-backed `cargo test -p <crate>`, run `bash .cursor/infra.sh` once — it brings up Docker, Postgres, and Redis in seconds because install baked the images and volumes. Pure-logic crate tests need nothing. Run `bash .cursor/stack.sh` for a product-ready environment.++After backend edits, run `bash .cursor/rebuild.sh`. Do not use `just stack update` with the read-only Nix binary directory.++No seeding or OTP is needed to log in: passwordless login auto-creates a user for any email, and the stack's auth service is built with `return_passwordless_code`, so the login API returns the code in its response (codes are also visible at the proxy's `/mailpit/` UI). `just seed-scenario apply --file seed/scenarios/team-perms.json` is optional, for multi-user team/permission fixtures. The `agent_harness_service` restart loop is expected without AI provider keys.++Leave `SQLX_OFFLINE` unset for `cargo test`. If SQLx reports missing cached query data, run `just prepare_db` instead of enabling offline mode. Run crate tests from the repository root with `cargo test -p <crate>`.diff --git a/docs/RUNNING_LOCALLY.md b/docs/RUNNING_LOCALLY.mdindex 1018192328..85e35e7104 100644--- a/docs/RUNNING_LOCALLY.md+++ b/docs/RUNNING_LOCALLY.md@@ -4,10 +4,13 @@ This guide explains how to run Macro on your machine. The local stack runs witho## What You Need-Install these tools before you start:+Install Nix before you start:1. [Nix](https://nix.dev/install-nix) package manager-2. [Docker](https://docs.docker.com/get-docker/) with the Compose v2 plugin (Docker Desktop, OrbStack, or Colima work)++On Linux, the Nix dev shell supplies the Docker CLI, daemon, Compose, and `fuse-overlayfs`. Nix is the only host dependency.++On macOS, install a Docker runtime such as Docker Desktop, OrbStack, or Colima. The Nix dev shell supplies the Docker CLI, but macOS still needs the runtime to provide the daemon.Clone the repository:@@ -18,7 +21,7 @@ cd macro## Enter the Nix Shell-The Nix shell provides `just`, Cargo, the Rust toolchain, Bun, sqlx, zig, and cargo-zigbuild. You do not need to install `just` or Cargo separately.+The Nix shell provides `just`, Cargo, the Rust toolchain, Bun, sqlx, zig, cargo-zigbuild, and Docker tooling. You do not need to install these tools separately.```bashnix developdiff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex e493a94675..b66ada8f7f 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -420,6 +420,36 @@}];+ localStackBinaryDefinitions = [+ {+ serviceName = "local-authentication";+ packageName = "authentication_service";+ binaries = [ "authentication_service" ];+ featureArgs = "--no-default-features --features return_passwordless_code";+ }+ {+ serviceName = "local-search-processing";+ packageName = "search_processing_service";+ binaries = [ "search_processing_service" ];+ featureArgs = "--no-default-features --features processing,service";+ }+ {+ serviceName = "local-upload-finalizer";+ packageName = "document_upload_finalizer_handler";+ binaries = [ "document_upload_finalizer_local_worker" ];+ }+ {+ serviceName = "local-agent-trigger";+ packageName = "agent_trigger_service";+ binaries = [ "agent_trigger_service" ];+ }+ {+ serviceName = "local-seed-cli";+ packageName = "seed_cli";+ binaries = [ "seed_cli" ];+ }+ ];+deployBinaryCargoExtraArgs ="--locked "+ pkgs.lib.concatMapStringsSep " " (@@ -446,6 +476,7 @@serviceName,packageName,binaries,+ featureArgs ? "",}:craneLib.buildPackage (commonArgs@@ -458,7 +489,8 @@doCheck = false;cargoExtraArgs ="--locked --package ${packageName} "- + pkgs.lib.concatMapStringsSep " " (binary: "--bin ${binary}") binaries;+ + pkgs.lib.concatMapStringsSep " " (binary: "--bin ${binary}") binaries+ + pkgs.lib.optionalString (featureArgs != "") " ${featureArgs}";CARGO_PROFILE = "release";installPhaseCommand = ''mkdir -p $out/bin@@ -477,6 +509,36 @@}) deployServiceBinaryDefinitions);+ localStackBinaryPackages = pkgs.lib.listToAttrs (+ map (def: {+ name = "local-stack-binaries-${def.serviceName}";+ value = deployServiceBinaryPackage def;+ }) localStackBinaryDefinitions+ );++ localStackDeployServiceNames = [+ "agent-harness-service"+ "connection-gateway"+ "contacts-service"+ "document-cognition-service"+ "document-storage-service"+ "email-service"+ "image-proxy-service"+ "notification-service"+ "static-file-service"+ "unfurl-service"+ ];++ localStackBinaries = pkgs.buildEnv {+ name = "local-stack-binaries";+ pathsToLink = [ "/bin" ];+ paths =+ pkgs.lib.attrValues localStackBinaryPackages+ ++ map (+ serviceName: deployServiceBinaryPackages."deploy-service-binaries-${serviceName}"+ ) localStackDeployServiceNames;+ };+# ── Lambda builds (crane + cargo-zigbuild) ─────────────────────# SPIKE: build a Rust Lambda handler reproducibly under nix/crane so# lambdas ride the same content-addressed cache as the service binaries@@ -832,7 +894,11 @@rustToolchainpython3]- ++ pkgs.lib.optionals isLinux [ mold ];+ ++ pkgs.lib.optionals isLinux [+ mold+ docker+ fuse-overlayfs+ ];in{@@ -913,7 +979,10 @@}// dopplerConfigBinPackages// deployServiceBinaryPackages- // deployLambdaPackages;+ // deployLambdaPackages+ // pkgs.lib.optionalAttrs isLinux {+ local-stack-binaries = localStackBinaries;+ };devShells = {default =diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rsindex 600aef5e72..2f6861c687 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/mod.rs@@ -38,6 +38,7 @@ mod ensure_daytona_snapshot;mod path_validation;mod preview_fly;mod pulumi_preview_pr;+mod push_local_stack_binaries;mod reusable_deploy_service;mod reusable_preview_service;mod runners;@@ -238,6 +239,11 @@ const WORKFLOWS: &[WorkflowFile] = &[file_name: "ensure_daytona_snapshot.yml",render_yaml: || render_gh_workflow(ensure_daytona_snapshot::ensure_daytona_snapshot)(),},+ WorkflowFile {+ slug: "push_local_stack_binaries",+ file_name: "push_local_stack_binaries.yml",+ render_yaml: || render_gh_workflow(push_local_stack_binaries::push_local_stack_binaries)(),+ },WorkflowFile {slug: "pulumi_preview_pr",file_name: "pulumi_preview_pr.yml",diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/push_local_stack_binaries.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/push_local_stack_binaries.rsnew file mode 100644index 0000000000..1d6e050fae--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/push_local_stack_binaries.rs@@ -0,0 +1,64 @@+//! Publish `.#local-stack-binaries` to the private S3 Nix cache.+//!+//! Sibling of the deploy pipeline on purpose: a cache miss or a failed `nix+//! copy` must not hold or fail `deploy_on_push` / `release-production`. The+//! ten deploy packages this aggregate reuses are already pushed by+//! `build-service-binaries`; this workflow exists for the five local-only+//! closures and the `buildEnv` wrapper. Generated into+//! `push_local_stack_binaries.yml`.++use gh_workflow::{+ Concurrency, Event, Expression, Job, Level, Permissions, Push, Workflow, WorkflowDispatch,+};++use crate::workflows::{runners, steps};++/// Build the workflow.+pub fn push_local_stack_binaries() -> Workflow {+ let mut push = Push::default().add_branch("main");+ for path in [+ xtask_paths::repo_glob!("Cargo.toml"),+ xtask_paths::repo_glob!("Cargo.lock"),+ xtask_paths::repo_glob!("rust-toolchain.toml"),+ xtask_paths::repo_glob!("crates/**"),+ xtask_paths::repo_glob!("services/**"),+ xtask_paths::repo_glob!("nix/**"),+ xtask_paths::repo_glob!("flake.nix"),+ xtask_paths::repo_glob!("flake.lock"),+ xtask_paths::repo_glob!(".github/actions/setup-nix/**"),+ xtask_paths::repo_glob!(".github/actions/teardown-nix/**"),+ xtask_paths::repo_glob!(".github/workflows/push_local_stack_binaries.yml"),+ ] {+ push = push.add_path(path);+ }++ Workflow::new("Push local stack binaries")+ .permissions(Permissions {+ contents: Some(Level::Read),+ ..Default::default()+ })+ .on(Event::default()+ .push(push)+ .workflow_dispatch(WorkflowDispatch::default()))+ .concurrency(+ Concurrency::new(Expression::new("${{ github.workflow }}-${{ github.ref }}"))+ .cancel_in_progress(true),+ )+ .add_job("push", push_job())+}++fn push_job() -> Job {+ Job::default()+ .name("Build and push local-stack-binaries")+ .runs_on(runners::Runner::Mid.to_string())+ .add_step(steps::checkout_v4())+ .add_step(steps::mount_nix_cache_volume())+ .add_step(steps::setup_nix_with_cache())+ .add_step(steps::nix_build(+ "Build local stack binaries",+ "\".#local-stack-binaries\"",+ "Local stack binaries realised into /nix/store.",+ ))+ .add_step(steps::push_nix_cache(".#local-stack-binaries"))+ .add_step(steps::teardown_nix())+}