Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf: bake a complete local stack for Cursor Cloud agents (#5770)
teo4afaf3df2e50
diff --git a/.cursor/cloud-lib.sh b/.cursor/cloud-lib.shnew file mode 100755index 0000000000..81f4b8d3b5--- /dev/null+++ b/.cursor/cloud-lib.sh@@ -0,0 +1,244 @@+# Shared by install.sh / start.sh / stack.sh. Sourced, not executed.+# Persistent caches live under $HOME so they survive `git checkout` into /workspace.++WORKSPACE_ROOT='/workspace'+CACHE_ROOT="${HOME}/.cache/macro-cloud"+TARGET_CACHE="${CACHE_ROOT}/target"+FRONTEND_CACHE="${CACHE_ROOT}/frontend"+export MACRO_STACK_SNAPSHOT_DIR="${CACHE_ROOT}/stack-snapshots"+STACK_SNAPSHOT_IMAGE='macro-cloud-stack-snapshot:prepared'++LOG_DIR="${HOME}/.cursor-cloud"+MACRODB_URL='postgres://user:password@localhost:5432/macrodb'+DOCKER_SOCK='/var/run/docker.sock'+DOCKER_IPTABLES_BACKEND='/usr/sbin/iptables-legacy'+DOCKER_IP6TABLES_BACKEND='/usr/sbin/ip6tables-legacy'+NIX_BIN='/nix/var/nix/profiles/default/bin/nix'+NIX_SOCK='/nix/var/nix/daemon-socket/socket'+export DATABASE_URL="${MACRODB_URL}"++mkdir -p "${LOG_DIR}" "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"++ensure_nix_installed() {+ if [ -x "${NIX_BIN}" ]; then+ return 0+ fi+ echo "cursor-cloud: installing Determinate Nix"+ curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \+ https://install.determinate.systems/nix |+ sudo sh -s -- install linux \+ --init none \+ --no-confirm \+ --extra-conf "trusted-users = root $(id -un)" \+ --extra-conf "sandbox = false"+ test -x "${NIX_BIN}"+}++ensure_nix_daemon() {+ ensure_nix_installed+ if "${NIX_BIN}" ping-store >/dev/null 2>&1; then+ return 0+ fi+ # Disk snapshots preserve socket path entries, not the process listening on+ # them. Remove the stale entry only after a real store probe has failed.+ sudo rm -f "${NIX_SOCK}"+ : >"${LOG_DIR}/nix-daemon.log"+ sudo setsid "${NIX_BIN}" daemon >>"${LOG_DIR}/nix-daemon.log" 2>&1 </dev/null &+ local n=0+ while [ "${n}" -lt 30 ]; do+ if "${NIX_BIN}" ping-store >/dev/null 2>&1; then+ return 0+ fi+ n=$((n + 1))+ sleep 1+ done+ echo "nix-daemon did not become ready" >&2+ sed -n '1,160p' "${LOG_DIR}/nix-daemon.log" >&2 || true+ return 1+}++ensure_docker_iptables_backend() {+ local desired_backend+ desired_backend="$(readlink -f "${DOCKER_IPTABLES_BACKEND}")"+ local desired_ip6_backend+ desired_ip6_backend="$(readlink -f "${DOCKER_IP6TABLES_BACKEND}")"+ local current_backend+ current_backend="$(readlink -f /etc/alternatives/iptables 2>/dev/null || true)"++ test -x "${DOCKER_IPTABLES_BACKEND}"+ test -x "${DOCKER_IP6TABLES_BACKEND}"++ # The cloud image can preserve a legacy FORWARD DROP policy while apt selects+ # iptables-nft. Make Docker program the enforcing legacy table instead of+ # opening FORWARD or bypassing Docker's per-network isolation chains.+ if [ "${current_backend}" != "${desired_backend}" ]; then+ sudo /usr/bin/update-alternatives \+ --set iptables "${DOCKER_IPTABLES_BACKEND}"+ fi+ if [ "$(readlink -f /etc/alternatives/ip6tables 2>/dev/null || true)" \+ != "${desired_ip6_backend}" ]; then+ sudo /usr/bin/update-alternatives \+ --set ip6tables "${DOCKER_IP6TABLES_BACKEND}"+ fi++ current_backend="$(readlink -f /etc/alternatives/iptables)"+ local current_ip6_backend+ current_ip6_backend="$(readlink -f /etc/alternatives/ip6tables)"+ test "${current_backend}" = "${desired_backend}"+ test "${current_ip6_backend}" = "${desired_ip6_backend}"+}++ensure_dockerd() {+ ensure_docker_iptables_backend+ local storage_driver='vfs'+ local storage_args=()+ if [ -x /usr/bin/fuse-overlayfs ]; then+ storage_driver='fuse-overlayfs'+ fi+ if ! sudo /usr/bin/grep -q '"storage-driver"[[:space:]]*:' /etc/docker/daemon.json 2>/dev/null; then+ storage_args+=("--storage-driver=${storage_driver}")+ fi+ if [ -S "${DOCKER_SOCK}" ]; then+ sudo chmod 666 "${DOCKER_SOCK}"+ if docker info >/dev/null 2>&1; then+ return 0+ fi+ fi+ # As with Nix, a snapshot can contain a dead Unix socket. Docker does not+ # reliably replace it while starting, so clear it after the API probe fails.+ sudo rm -f "${DOCKER_SOCK}"+ : >"${LOG_DIR}/dockerd.log"+ sudo setsid /usr/bin/dockerd "${storage_args[@]}" \+ >>"${LOG_DIR}/dockerd.log" 2>&1 </dev/null &+ local n=0+ while [ "${n}" -lt 60 ]; do+ if [ -S "${DOCKER_SOCK}" ]; then+ sudo chmod 666 "${DOCKER_SOCK}"+ if docker info >/dev/null 2>&1; then+ return 0+ fi+ fi+ n=$((n + 1))+ sleep 1+ done+ echo "dockerd did not become ready" >&2+ sed -n '1,160p' "${LOG_DIR}/dockerd.log" >&2 || true+ return 1+}++in_pinned_nix_shell() {+ [ "${MACRO_CLOUD_PINNED_NIX:-}" = "1" ]+}++reenter_pinned_nix_shell() {+ local script_path="$1"+ shift+ exec "${NIX_BIN}" develop "${WORKSPACE_ROOT}" --command /usr/bin/env \+ MACRO_CLOUD_PINNED_NIX=1 /usr/bin/bash "${script_path}" "$@"+}++workspace_owner() {+ stat -c '%U' "${WORKSPACE_ROOT}"+}++workspace_group() {+ stat -c '%G' "${WORKSPACE_ROOT}"+}++remove_workspace_target_mounts() {+ command -v docker >/dev/null 2>&1 || return 0+ docker info >/dev/null 2>&1 || return 0+ local id source+ for id in $(docker ps -aq); do+ while IFS= read -r source; do+ case "${source}" in+ "${WORKSPACE_ROOT}/target" | "${WORKSPACE_ROOT}/target"/*)+ echo "cursor-cloud: removing leftover container ${id} mounted on ${source}"+ docker rm -f "${id}" >/dev/null 2>&1 || true+ break+ ;;+ esac+ done < <(docker inspect --format '{{range .Mounts}}{{.Source}}{{println}}{{end}}' "${id}")+ done+}++# Move /workspace/target into $HOME (same filesystem: rename). Recreate the+# symlink after every checkout so Cargo's incremental cache survives it.+ensure_persistent_caches() {+ mkdir -p "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"+ remove_workspace_target_mounts++ if [ -e "${WORKSPACE_ROOT}/target" ] && [ ! -L "${WORKSPACE_ROOT}/target" ]; then+ sudo chown -R "$(workspace_owner):$(workspace_group)" "${WORKSPACE_ROOT}/target"+ if [ -x "${WORKSPACE_ROOT}/target/x86_64-unknown-linux-gnu/debug/document_storage_service" ] \+ && [ ! -x "${TARGET_CACHE}/x86_64-unknown-linux-gnu/debug/document_storage_service" ]; then+ echo "cursor-cloud: moving ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"+ rm -rf "${TARGET_CACHE}"+ mv "${WORKSPACE_ROOT}/target" "${TARGET_CACHE}"+ else+ echo "cursor-cloud: replacing ${WORKSPACE_ROOT}/target with cache symlink"+ rm -rf "${WORKSPACE_ROOT}/target"+ fi+ fi++ mkdir -p "${TARGET_CACHE}"+ ln -sfn "${TARGET_CACHE}" "${WORKSPACE_ROOT}/target"+ sudo chown -R "$(workspace_owner):$(workspace_group)" "${TARGET_CACHE}"+ echo "cursor-cloud: ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"++ if [ ! -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ] && [ -f "${FRONTEND_CACHE}/index.html" ]; then+ mkdir -p "${WORKSPACE_ROOT}/apps/web/dist"+ cp -a "${FRONTEND_CACHE}/." "${WORKSPACE_ROOT}/apps/web/dist/"+ echo "cursor-cloud: restored frontend bundle from cache"+ elif [ -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ]; then+ cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"+ fi++ if [ -d "${WORKSPACE_ROOT}/infra/local/generated/.snapshots" ]; then+ cp -a "${WORKSPACE_ROOT}/infra/local/generated/.snapshots/." "${MACRO_STACK_SNAPSHOT_DIR}/"+ fi+}++bake_stack_snapshot_image() {+ local key="$1"+ local snapshot_dir="$2"++ test -f "${snapshot_dir}/manifest.json"+ docker build \+ --tag "${STACK_SNAPSHOT_IMAGE}" \+ --file - \+ "${snapshot_dir}" <<EOF+FROM alpine:3+LABEL com.macro.stack-snapshot-key="${key}"+COPY . /snapshot/+EOF+ docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null+}++restore_stack_snapshot_image() {+ if ! docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null 2>&1; then+ echo "cursor-cloud: prepared stack snapshot image not found"+ return 0+ fi++ local key+ key="$(docker image inspect \+ --format '{{ index .Config.Labels "com.macro.stack-snapshot-key" }}' \+ "${STACK_SNAPSHOT_IMAGE}")"+ case "${key}" in+ ''|*[!0-9a-f]*)+ echo "prepared stack snapshot image has invalid key: ${key}" >&2+ return 1+ ;;+ esac++ local destination="${MACRO_STACK_SNAPSHOT_DIR}/${key}"+ mkdir -p "${destination}"+ docker run --rm \+ --user "$(id -u):$(id -g)" \+ --volume "${destination}:/restore" \+ "${STACK_SNAPSHOT_IMAGE}" \+ sh -ceu 'cp -R /snapshot/. /restore/'+ test -f "${destination}/manifest.json"+ echo "cursor-cloud: restored stack snapshot ${key} from image"+}diff --git a/.cursor/environment.json b/.cursor/environment.jsonnew file mode 100644index 0000000000..632ad42a26--- /dev/null+++ b/.cursor/environment.json@@ -0,0 +1,5 @@+{+ "name": "macro",+ "install": "bash .cursor/install.sh",+ "start": "bash .cursor/start.sh"+}diff --git a/.cursor/install.sh b/.cursor/install.shnew file mode 100755index 0000000000..4e3ad3c359--- /dev/null+++ b/.cursor/install.sh@@ -0,0 +1,152 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++# shellcheck source=cloud-lib.sh+source "${SCRIPT_DIR}/cloud-lib.sh"++ensure_apt_packages() {+ local pkg+ local missing=()+ for pkg in fuse-overlayfs libssl-dev pkg-config postgresql-client; do+ if ! dpkg -s "${pkg}" >/dev/null 2>&1; then+ missing+=("${pkg}")+ fi+ done+ # Do not replace Docker CE on bases that already provide a complete engine.+ if [ ! -x /usr/bin/docker ] || [ ! -x /usr/bin/dockerd ]; then+ missing+=(docker.io docker-buildx docker-compose-v2)+ fi+ if [ "${#missing[@]}" -ne 0 ]; then+ sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq+ sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \+ -o Dpkg::Options::=--force-confdef \+ -o Dpkg::Options::=--force-confold \+ "${missing[@]}"+ fi+ test -x /usr/bin/dockerd+ /usr/bin/docker buildx version+ /usr/bin/docker compose version+}++crate_env_paths() {+ awk '+ $0 == "setup_test_envs:" { p = 1; next }+ p && /^[A-Za-z_]/ { exit }+ p && />>/ {+ sub(/^.*>>[[:space:]]*/, "")+ }+ ' /workspace/tooling/just/rust.just+}++# setup_test_envs always appends; write each DATABASE_URL at most once.+ensure_test_envs() {+ local f+ while IFS= read -r f; do+ [ -z "${f}" ] && continue+ mkdir -p "$(dirname "${f}")"+ if [ -f "${f}" ] && grep -qxF "DATABASE_URL=${MACRODB_URL}" "${f}"; then+ continue+ fi+ printf 'DATABASE_URL=%s\n' "${MACRODB_URL}" >>"${f}"+ done < <(crate_env_paths)+}++build_frontend_artifact() {+ bun install --frozen-lockfile+ (+ cd "${WORKSPACE_ROOT}/apps/web"+ MODE=development NODE_ENV=production VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin \+ bun run --bun build+ )+ test -f "${WORKSPACE_ROOT}/apps/web/dist/index.html"+ mkdir -p "${FRONTEND_CACHE}"+ cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"+}++cleanup_stack() {+ cd "${WORKSPACE_ROOT}"+ timeout --kill-after=10s 2m just --quiet stack down+}++cleanup_stack_best_effort() {+ cleanup_stack >/dev/null 2>&1 || true+}++prepare_durable_stack() {+ # `infra-only` is the existing finite CI bake mode: it materializes every+ # Compose image, initializes the real infra, and saves/restores the+ # content-addressed volume snapshot without starting secret-dependent apps.+ just stack up \+ --infra-only \+ --no-doppler \+ --build-aux-services \+ --json++ local snapshot_json+ snapshot_json="$(+ cargo run --quiet --manifest-path Cargo.toml -p xtask_local --features local-stack -- \+ stack snapshot --json+ )"+ local snapshot_fields=()+ mapfile -t snapshot_fields < <(python3 -c '+import json+import sys++snapshot = json.load(sys.stdin)+key = snapshot["key"]+if not snapshot["present"]:+ raise SystemExit(f"stack snapshot missing for key {key}")+print(key)+print(snapshot["dir"])+' <<<"${snapshot_json}")+ local snapshot_key="${snapshot_fields[0]}"+ local snapshot_dir="${snapshot_fields[1]}"++ bake_stack_snapshot_image "${snapshot_key}" "${snapshot_dir}"+ local image_id+ image_id="$(docker image inspect --format '{{.Id}}' "${STACK_SNAPSHOT_IMAGE}")"++ echo "cursor-cloud install: stack snapshot ${snapshot_key} at ${snapshot_dir}"+ echo "cursor-cloud install: stack snapshot image ${STACK_SNAPSHOT_IMAGE} (${image_id})"+}++if ! in_pinned_nix_shell; then+ ensure_apt_packages+ ensure_nix_daemon+ ensure_dockerd+ reenter_pinned_nix_shell "${SCRIPT_DIR}/install.sh" "$@"+fi++ensure_persistent_caches++docker pull pgvector/pgvector:pg18+docker pull redis/redis-stack:latest++cd "${WORKSPACE_ROOT}"+just run_dbs -d+trap cleanup_stack_best_effort EXIT++ensure_test_envs++just initialize_dbs+cargo fetch --locked+unset SQLX_OFFLINE+cargo test --no-run -p macro_db_client++echo "cursor-cloud install: test-ready"++# Build tools own cache validity: Bun, Cargo, and BuildKit compare the current+# source/lock/toolchain inputs instead of trusting surviving files or image tags.+build_frontend_artifact+echo "cursor-cloud install: frontend artifact"++# The bake primitive builds the runtime image and complete Rust binary inventory+# before initializing and snapshotting infra.+prepare_durable_stack+cleanup_stack+trap - EXIT++echo "cursor-cloud install: durable stack ready"diff --git a/.cursor/stack.sh b/.cursor/stack.shnew file mode 100755index 0000000000..871d4fdde0--- /dev/null+++ b/.cursor/stack.sh@@ -0,0 +1,25 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++# On-demand product stack. start.sh is infra-only so cargo-test agents do not+# boot FusionAuth. Cargo, Bun, and BuildKit incrementally reconcile the checked+# out branch against the durable main-build caches before services start.++# shellcheck source=cloud-lib.sh+source "${SCRIPT_DIR}/cloud-lib.sh"++if ! in_pinned_nix_shell; then+ ensure_nix_daemon+ ensure_dockerd+ reenter_pinned_nix_shell "${SCRIPT_DIR}/stack.sh" "$@"+fi++/usr/bin/bash "${SCRIPT_DIR}/start.sh"++cd "${WORKSPACE_ROOT}"+restore_stack_snapshot_image+just stack up --no-doppler --build-aux-services++echo "cursor-cloud stack: app ready"diff --git a/.cursor/start.sh b/.cursor/start.shnew file mode 100755index 0000000000..2eb37885f1--- /dev/null+++ b/.cursor/start.sh@@ -0,0 +1,22 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++# shellcheck source=cloud-lib.sh+source "${SCRIPT_DIR}/cloud-lib.sh"++if ! in_pinned_nix_shell; then+ ensure_nix_daemon+ ensure_dockerd+ reenter_pinned_nix_shell "${SCRIPT_DIR}/start.sh" "$@"+fi++ensure_persistent_caches++cd "${WORKSPACE_ROOT}"+just run_dbs -d++pg_isready -h 127.0.0.1 -p 5432++echo "cursor-cloud start: infra ready"diff --git a/.dockerignore b/.dockerignoreindex 581f164ed6..931cf6e3e1 100644--- a/.dockerignore+++ b/.dockerignore@@ -2,8 +2,10 @@.git.jj.agents+.cursor.claude/.local.pi/outputs+/infra/local/generated# Legacy layout directories may still contain ignored local build artifacts./agentsdiff --git a/CLAUDE.md b/CLAUDE.mdindex d5b2b9fe79..d6baf9a84e 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -78,13 +78,15 @@ just check # Type check without building### Testing```bash-# Setup test environment-docker compose --project-directory . -f docker/docker-compose.yml up -d postgres-just setup_test_envs # Setup .env files for tests-just initialize_dbs # Initialize all databases-just test # Run tests+just create_networks+just run_dbs -d+just setup_test_envs+just initialize_dbs+cargo test -p {crate}```+`just test` does not exist. Leave `SQLX_OFFLINE` unset when you run `cargo test`. Run `just prepare_db` only if you changed SQL queries.+### Pre Commit```bashcargo fmt # format@@ -93,12 +95,11 @@ just clippy # extra lints / best practices### Database Management-```bash-just setup_macrodb # Setup main database-just setup_commsdb # Setup communications database-just setup_emaildb # Setup email database-just setup_contactsdb # Setup contacts database-```+Use `just setup_macrodb` or `just initialize_dbs` to create and migrate MacroDB. Those recipes are the same.++Schemas live in `crates/macro_db_client/migrations/`.++To reset MacroDB, run `just crates/macro_db_client/drop_db -y -f`, then `just setup_macrodb`.### Lambda Building@@ -316,35 +317,12 @@ don't inject it directly into the error message.## Cursor Cloud specific instructions-This VM runs the intended local dev stack from `docs/RUNNING_LOCALLY.md` via Nix + Docker.-The startup layer (nix daemon, docker daemon, `bun install`) is handled by the environment-update script; the notes below are durable, non-obvious caveats for running the app here.--- Toolchain lives in the Nix dev shell. Run dev commands from the repo root as- `nix develop --command bash -c '<cmd>'`. If `nix` is not found, first- `. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh`. There is no systemd- (init is `tini`); `nix-daemon` and `dockerd` are plain background processes.-- Use the `--no-doppler` path (this repo is run without team secrets). Prefer the- headless orchestrator `just stack ...` over `just run_local` — `run_local` has an- interactive hotkey loop that does not suit a non-interactive agent.-- KEY GOTCHA (openssh): `just stack up` / `run_local` shell out to `ssh-keygen`/`ssh`- for the SDK webhook relay. The VM's `/exec-daemon/ssh-keygen` shim is first on PATH and- crashes under the Nix shell's `LD_LIBRARY_PATH` (glibc 2.42 vs system 2.39). Fix by- putting a self-consistent Nix openssh first on PATH. openssh is installed in the Nix- profile (`nix profile add nixpkgs#openssh`), so launch the stack as:- `nix develop --command bash -c 'export PATH=$HOME/.nix-profile/bin:$PATH; just stack up --no-doppler'`-- First `stack up` cross-compiles all service binaries with `cargo-zigbuild`- (slow — ~10 min cold on 4 cores); later runs reuse `sccache` and a cached init snapshot,- so they are fast. `just stack status` / `just stack update` / `just stack down` manage it.-- Endpoints (default instance): app at `http://localhost:8090/app/`, proxy at `:8090`,- Mailpit at `http://localhost:8090/mailpit/`, FusionAuth at `:9011`.-- Login is passwordless: the one-time code is delivered to Mailpit (not a real inbox).-- Seed sample data: `just seed-scenario apply --file seed/scenarios/team-perms.json`- (personas `alice`..`eve` at `<name>@seed.macro.local`). The login links it prints embed- port `3000` (the `run_local` dev-server default); in headless `stack` mode use `:8090`,- e.g. `http://localhost:8090/app/[email protected]`.-- `agent_harness_service` restarts continuously in a `--no-doppler` stack (it needs AI- provider keys); this is expected and does not affect non-AI features.-- Tests run against the live local Postgres — do not set `SQLX_OFFLINE=true` for- `cargo test` (see the sqlx notes above). For DB-backed crate tests, run- `just setup_test_envs` first; pure-logic crates (e.g. `cargo test -p email_utils`) need no setup.+Environment scripts are `.cursor/install.sh` (durable files), `.cursor/start.sh` (dockerd, then Postgres and Redis), and `.cursor/stack.sh` (on-demand product stack).++After boot, Postgres should already be up. If `:5432` is closed, run `bash .cursor/start.sh`.++Cargo `target/`, the frontend bundle, and the stack init snapshot live under `$HOME/.cache/macro-cloud` and are symlinked or copied back into `/workspace`. A git checkout wipes `/workspace/target`. Cargo and BuildKit decide whether the next bake is incremental.++Infra-ready is enough for `cargo test -p`. Product-ready is `bash .cursor/stack.sh`, which runs `just stack up --no-doppler --build-aux-services`. Install always bakes with `--infra-only --build-aux-services`. The pinned nix shell puts OpenSSH on PATH so `ssh-keygen` is not `/exec-daemon/ssh-keygen`.++Do not follow the old `docker compose -f docker/docker-compose.yml up -d postgres` line. It skips `create_networks` and Redis.diff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex 9fee5b771a..a696bd4290 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -315,13 +315,14 @@ services:# Agent session trigger events -> sandboxed coding agent sessions, plus the# session control routes, which have to be served from the process that owns# the live transports they act on.- # Starts credential-less (and warns); arm it with+ # Starts when both credentials are supplied:# `DAYTONA_API_KEY=... GITHUB_TOKEN=... just run_local`.# ============================================================================agent_harness_service:<<: [*common-env, *rust-services-image]command: ["/app/out/agent_harness_service"]restart: on-failure+ profiles: ["agent-harness"]# No host port: everything reaches this through the proxy, and a second# local instance would collide on a fixed publish.environment:@@ -458,7 +459,13 @@ services:- sync-servicedatabases:healthcheck:- test: ["CMD", "curl", "-f", "http://localhost:8787/health"]+ test:+ [+ "CMD",+ "node",+ "-e",+ "fetch('http://localhost:8787/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",+ ]interval: 30stimeout: 10sretries: 3diff --git a/docker/lexical-service.Dockerfile.dockerignore b/docker/lexical-service.Dockerfile.dockerignoreindex a431e4ddcf..5655c42d9c 100644--- a/docker/lexical-service.Dockerfile.dockerignore+++ b/docker/lexical-service.Dockerfile.dockerignore@@ -2,8 +2,10 @@.git.jj.agents+.cursor.claude/.local.pi/outputs+/infra/local/generated# Legacy layout directories may still contain ignored local build artifacts./agentsdiff --git a/docker/sync-service.Dockerfile b/docker/sync-service.Dockerfileindex b010e5be85..254d93bea9 100644--- a/docker/sync-service.Dockerfile+++ b/docker/sync-service.Dockerfile@@ -1,3 +1,5 @@+# syntax=docker/dockerfile:1.7+# Builder: compiles the worker (Rust -> wasm + JS shim). Nothing from this# stage's toolchain (rustc, clang, npm dev deps, the cargo target dir) is# needed at runtime, so it all stays here.@@ -41,19 +43,18 @@ RUN cd services/sync-service && npm ci# Copy bebop schema and generate typescript bindingsRUN cd services/sync-service/bebop && npx bebopc build-# Build the actual application-RUN cd services/sync-service && worker-build --profile sync-service-release+# Build the actual application. The workspace context changes whenever a+# dependent Rust crate changes; preserve Cargo's target cache across those+# BuildKit layer invalidations so branch reconciliation stays incremental.+RUN --mount=type=cache,id=macro-sync-cargo-registry,target=/usr/local/cargo/registry \+ --mount=type=cache,id=macro-sync-cargo-target,target=/app/target \+ cd services/sync-service && worker-build --profile sync-service-release# Runtime: wrangler dev serving the prebuilt worker. Needs node + the# production npm deps (wrangler brings miniflare/workerd along), the built# worker, the wrangler config, and the D1 migrations — nothing else.FROM node:22-bookworm-slim-# curl for the compose healthcheck; ca-certificates for wrangler's TLS.-RUN apt-get update && \- apt-get install -y --no-install-recommends curl ca-certificates && \- rm -rf /var/lib/apt/lists/*-WORKDIR /appCOPY services/sync-service/package.json services/sync-service/package-lock.json ./@@ -67,4 +68,4 @@ EXPOSE 8787# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,# apply local D1 migrations, then start.-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\nDSS_INTERNAL_AUTH_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" "$DSS_INTERNAL_AUTH_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'+CMD ["sh", "-c", "printf \"LOCAL_API_KEY=%s\\nDOCUMENT_PERMISSIONS_SECRET=%s\\nSPS_API_SECRET_KEY=%s\\nDSS_INTERNAL_AUTH_KEY=%s\\n\" \"$INTERNAL_API_SECRET_KEY\" \"$DOCUMENT_PERMISSIONS_SECRET\" \"$INTERNAL_API_SECRET_KEY\" \"$DSS_INTERNAL_AUTH_KEY\" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml"]diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex fe80c8ce04..e493a94675 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -769,6 +769,7 @@paralleldocker-composecurl+ opensshwgetkcatxzdiff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rsindex 57ac786768..9cf80e2279 100644--- a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs@@ -17,25 +17,26 @@ pub const RUNTIME_IMAGE_TAG: &str = "macro-local-runtime:dev";const DOCKERFILE_REL: &str = "docker/Dockerfile.runtime";-/// Build the runtime image for the host arch if it is missing (or `force`).+/// Reconcile the runtime image for the host arch through BuildKit.+///+/// BuildKit's content cache makes an unchanged build cheap while still+/// invalidating the image when the Dockerfile or build context changes.+/// Existence alone is not a valid cache key because the tag survives branch+/// checkouts. `force` disables BuildKit's layer cache.pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Result<()> {- if !force && image_exists() {- return Ok(());- }let ws = workspace_root();let mut cmd = Command::new("docker");cmd.current_dir(&ws).args(["buildx", "build", "--platform", target.docker_platform]).args(["-f", DOCKERFILE_REL]).args(["-t", RUNTIME_IMAGE_TAG])- .arg("--load")- .arg(".");+ .arg("--load");+ if force {+ cmd.arg("--no-cache");+ }+ cmd.arg(".");stage.run(&format!("Building runtime image {RUNTIME_IMAGE_TAG}"),&mut cmd,)}--fn image_exists() -> bool {- super::super::docker::image_exists(RUNTIME_IMAGE_TAG)-}diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rsindex 4d207609e8..168bd88b32 100644--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs@@ -96,7 +96,7 @@ pub struct BuildArgs {/// Skip building; reuse whatever is in the target dir.#[arg(long)]pub no_build: bool,- /// Rebuild Docker-built auxiliary services (sync, websocket, lexical).+ /// Rebuild every repository-built local Docker service.#[arg(long)]pub build_aux_services: bool,/// Use this dir as the `/app/out` source instead of building.@@ -175,7 +175,7 @@ pub struct ValidateEnvArgs {#[derive(Args, Clone, Default)]pub struct ForceArg {- /// Rebuild even if the image already exists.+ /// Rebuild without BuildKit's layer cache.#[arg(long)]pub force: bool,}diff --git a/tooling/xtask/crates/xtask_local/src/local/docker.rs b/tooling/xtask/crates/xtask_local/src/local/docker.rsindex 4baaefc0ab..895a13cb81 100644--- a/tooling/xtask/crates/xtask_local/src/local/docker.rs+++ b/tooling/xtask/crates/xtask_local/src/local/docker.rs@@ -1,8 +1,7 @@//! Thin bollard-backed helpers for the discrete Docker Engine operations the-//! orchestrator needs outside of `docker compose` — checking whether the-//! runtime image exists and idempotently ensuring the per-instance external-//! networks/volumes. (Compose orchestration itself stays on the CLI; bollard-//! has no compose support.)+//! orchestrator needs outside of `docker compose` — idempotently ensuring the+//! per-instance external networks/volumes. (Compose orchestration itself stays+//! on the CLI; bollard has no compose support.)use anyhow::{Context, Result};use bollard::Docker;@@ -32,15 +31,6 @@ fn connect() -> Result<Docker> {}}-/// Whether an image with `tag` exists locally.-pub fn image_exists(tag: &str) -> bool {- block_on(async {- let docker = connect()?;- Ok::<_, anyhow::Error>(docker.inspect_image(tag).await.is_ok())- })- .unwrap_or(false)-}-/// Idempotently create a bridge network (no-op if it already exists).pub fn ensure_network(name: &str) -> Result<()> {block_on(async {diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rsindex cdbcfd1dcf..681dce0317 100644--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs@@ -11,6 +11,7 @@//! Design invariant: Docker never compiles Rust. Binaries are built on the host//! and bind-mounted read-only into the runtime image at `/app/out`.+use std::collections::BTreeMap;use std::path::PathBuf;pub mod arch;@@ -157,7 +158,36 @@ use anyhow::Result;use instance::Instance;use stage::Stage;-const AUX_SERVICE_IMAGES: &[&str] = &["websocket_service", "sync_service", "lexical_service"];+/// Every non-Rust local service whose image is built from this repository.+/// `docker compose up` builds a missing image implicitly, but an Environment+/// Build must materialize all of them so a fresh agent never discovers one at+/// stack-start time.+const LOCAL_BUILD_SERVICE_IMAGES: &[&str] = &[+ "websocket_service",+ "sync_service",+ "lexical_service",+ "ai_editing_worker",+ "analytics_proxy",+ "sdk-webhook-relay",+ "search",+];++/// Repository-built app containers safe to recreate during `stack update`.+/// OpenSearch is built for cold-stack correctness but remains under the infra+/// lifecycle, which waits for health before Rust services can reconnect.+const LOCAL_RECREATE_SERVICE_IMAGES: &[&str] = &[+ "websocket_service",+ "sync_service",+ "lexical_service",+ "ai_editing_worker",+ "analytics_proxy",+ "sdk-webhook-relay",+];++/// Image-only app services that infra-only bake mode does not otherwise start.+/// Infra images are pulled by `bring_up_infra`; the Rust runtime image is built+/// separately by [`build::ensure_runtime_image`].+const LOCAL_PULL_SERVICE_IMAGES: &[&str] = &["proxy", "mailpit", "static_file_cdn"];/// The artifact-driven preview updater can refresh every Docker-built app/// service. Keep this separate so enabling local aux rebuilds does not make the@@ -210,7 +240,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {// Foreground: resolve env, build binaries + runtime image, generate the// compose override / Caddyfile / kickstart. None of this touches the volumes// or containers the teardown is removing, so it's safe to overlap.- let (env, target) = prepare(&stage, mode, &instance, args, false)?;+ let (env, target) = prepare(&stage, mode, &instance, args, false, false)?;// Join the background teardown before we (re)create volumes + bring infra up,// surfaced as a live spinner so it's clear what we're blocked on. It@@ -474,6 +504,7 @@ fn prepare(instance: &Instance,args: &cli::RunArgs,static_frontend: bool,+ pull_app_images: bool,) -> Result<(env_layer::ResolvedEnv, arch::Target)> {let env = env_layer::resolve(mode,@@ -516,6 +547,9 @@ fn prepare(if args.build.build_aux_services {build_aux_service_images(stage, instance, &env)?;}+ if pull_app_images {+ pull_app_service_images(stage, instance, &env)?;+ }Ok((env, target))}@@ -523,7 +557,17 @@ fn prepare(/// generated env. Shared by bring-up and the binary-reload restart.fn compose_cmd(instance: &Instance, env: &env_layer::ResolvedEnv) -> Command {let files = gen_compose::compose_files(instance);- gen_compose::docker_compose(instance, &files, &env.generated_path)+ let mut command = gen_compose::docker_compose(instance, &files, &env.generated_path);+ if agent_harness_enabled(&env.merged) {+ command.args(["--profile", "agent-harness"]);+ }+ command+}++fn agent_harness_enabled(env: &BTreeMap<String, String>) -> bool {+ ["DAYTONA_API_KEY", "GITHUB_TOKEN"]+ .iter()+ .all(|key| env.get(*key).is_some_and(|value| !value.trim().is_empty()))}fn build_aux_service_images(@@ -532,10 +576,20 @@ fn build_aux_service_images(env: &env_layer::ResolvedEnv,) -> Result<()> {let mut build = compose_cmd(instance, env);- build.arg("build").args(AUX_SERVICE_IMAGES);+ build.arg("build").args(LOCAL_BUILD_SERVICE_IMAGES);stage.run("Building auxiliary service images", &mut build)}+fn pull_app_service_images(+ stage: &Stage,+ instance: &Instance,+ env: &env_layer::ResolvedEnv,+) -> Result<()> {+ let mut pull = compose_cmd(instance, env);+ pull.arg("pull").args(LOCAL_PULL_SERVICE_IMAGES);+ stage.run("Pulling image-only app services", &mut pull)+}+fn recreate_aux_service_containers(stage: &Stage,instance: &Instance,@@ -543,7 +597,7 @@ fn recreate_aux_service_containers() -> Result<()> {let mut up = compose_cmd(instance, env);up.args(["up", "-d", "--force-recreate", "--no-deps"])- .args(AUX_SERVICE_IMAGES);+ .args(LOCAL_RECREATE_SERVICE_IMAGES);stage.run("Recreating auxiliary service containers", &mut up)}diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rsindex 175e9e6a62..f6dc0f4aa4 100644--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs@@ -73,7 +73,7 @@ pub struct UpdateArgs {/// Apply this complete prebuilt binary set instead of invoking Cargo.#[arg(long, conflicts_with = "build_aux_services")]pub binaries_dir: Option<PathBuf>,- /// Rebuild the Docker-built auxiliary services (sync, websocket, lexical).+ /// Rebuild every repository-built local Docker service.#[arg(long)]pub build_aux_services: bool,/// Recreate registry-refreshed preview auxiliary services without building.@@ -180,7 +180,14 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {});let static_frontend = !args.run.no_frontend && !args.infra_only;- let (env, target) = super::prepare(&stage, mode, &instance, &args.run, static_frontend)?;+ let (env, target) = super::prepare(+ &stage,+ mode,+ &instance,+ &args.run,+ static_frontend,+ args.infra_only,+ )?;let configured_binaries = args.run.builddiff --git a/tooling/xtask/crates/xtask_local/src/local/test.rs b/tooling/xtask/crates/xtask_local/src/local/test.rsindex 99d6812947..4bfe836257 100644--- a/tooling/xtask/crates/xtask_local/src/local/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/test.rs@@ -1,4 +1,5 @@use super::*;+use std::collections::{BTreeMap, BTreeSet};/// Every mode whose spec we assert invariants over.const MODES: &[Mode] = &[Mode::Local, Mode::Dev];@@ -25,3 +26,60 @@ fn mode_specs_are_coherent() {);}}++#[test]+fn durable_bake_covers_every_repository_built_local_image() {+ fn direct_build_services(path: &std::path::Path) -> BTreeSet<String> {+ let raw = std::fs::read_to_string(path).unwrap();+ let compose: serde_yaml::Value = serde_yaml::from_str(&raw).unwrap();+ compose["services"]+ .as_mapping()+ .unwrap()+ .iter()+ .filter_map(|(name, service)| {+ service+ .as_mapping()+ .is_some_and(|service| service.contains_key("build"))+ .then(|| name.as_str().unwrap().to_string())+ })+ .collect()+ }++ let mut expected = direct_build_services(&repo_root().join("docker/docker-compose.yml"));+ expected.extend(direct_build_services(+ &repo_root().join("docker/docker-compose-databases.yml"),+ ));+ // The generated override adds this build definition.+ expected.insert("sdk-webhook-relay".to_string());+ // The Rust services use the host-built runtime image; this profile-only+ // helper is intentionally never needed by local stack preparation.+ expected.remove("rust_services_image");+ for service in inventory::RUST_SERVICES {+ expected.remove(service.compose_name);+ }++ let actual: BTreeSet<_> = LOCAL_BUILD_SERVICE_IMAGES+ .iter()+ .map(|name| name.to_string())+ .collect();+ assert_eq!(actual, expected);+ assert_eq!(+ LOCAL_PULL_SERVICE_IMAGES,+ ["proxy", "mailpit", "static_file_cdn"]+ );+}++#[test]+fn agent_harness_requires_both_credentials() {+ let mut env = BTreeMap::new();+ assert!(!agent_harness_enabled(&env));++ env.insert("DAYTONA_API_KEY".into(), "daytona".into());+ assert!(!agent_harness_enabled(&env));++ env.insert("GITHUB_TOKEN".into(), "github".into());+ assert!(agent_harness_enabled(&env));++ env.insert("DAYTONA_API_KEY".into(), " ".into());+ assert!(!agent_harness_enabled(&env));+}