Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf: bake a complete local stack for Cursor Cloud agents (#5770)

teo4afaf3df2e50
diff --git a/.cursor/cloud-lib.sh b/.cursor/cloud-lib.sh
new file mode 100755
index 0000000000..81f4b8d3b5
--- /dev/null
+++ b/.cursor/cloud-lib.sh
@@ -0,0 +1,244 @@
+# Shared by install.sh / start.sh / stack.sh. Sourced, not executed.
+# Persistent caches live under $HOME so they survive `git checkout` into /workspace.
+
+WORKSPACE_ROOT='/workspace'
+CACHE_ROOT="${HOME}/.cache/macro-cloud"
+TARGET_CACHE="${CACHE_ROOT}/target"
+FRONTEND_CACHE="${CACHE_ROOT}/frontend"
+export MACRO_STACK_SNAPSHOT_DIR="${CACHE_ROOT}/stack-snapshots"
+STACK_SNAPSHOT_IMAGE='macro-cloud-stack-snapshot:prepared'
+
+LOG_DIR="${HOME}/.cursor-cloud"
+MACRODB_URL='postgres://user:password@localhost:5432/macrodb'
+DOCKER_SOCK='/var/run/docker.sock'
+DOCKER_IPTABLES_BACKEND='/usr/sbin/iptables-legacy'
+DOCKER_IP6TABLES_BACKEND='/usr/sbin/ip6tables-legacy'
+NIX_BIN='/nix/var/nix/profiles/default/bin/nix'
+NIX_SOCK='/nix/var/nix/daemon-socket/socket'
+export DATABASE_URL="${MACRODB_URL}"
+
+mkdir -p "${LOG_DIR}" "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"
+
+ensure_nix_installed() {
+ if [ -x "${NIX_BIN}" ]; then
+ return 0
+ fi
+ echo "cursor-cloud: installing Determinate Nix"
+ curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \
+ https://install.determinate.systems/nix |
+ sudo sh -s -- install linux \
+ --init none \
+ --no-confirm \
+ --extra-conf "trusted-users = root $(id -un)" \
+ --extra-conf "sandbox = false"
+ test -x "${NIX_BIN}"
+}
+
+ensure_nix_daemon() {
+ ensure_nix_installed
+ if "${NIX_BIN}" ping-store >/dev/null 2>&1; then
+ return 0
+ fi
+ # Disk snapshots preserve socket path entries, not the process listening on
+ # them. Remove the stale entry only after a real store probe has failed.
+ sudo rm -f "${NIX_SOCK}"
+ : >"${LOG_DIR}/nix-daemon.log"
+ sudo setsid "${NIX_BIN}" daemon >>"${LOG_DIR}/nix-daemon.log" 2>&1 </dev/null &
+ local n=0
+ while [ "${n}" -lt 30 ]; do
+ if "${NIX_BIN}" ping-store >/dev/null 2>&1; then
+ return 0
+ fi
+ n=$((n + 1))
+ sleep 1
+ done
+ echo "nix-daemon did not become ready" >&2
+ sed -n '1,160p' "${LOG_DIR}/nix-daemon.log" >&2 || true
+ return 1
+}
+
+ensure_docker_iptables_backend() {
+ local desired_backend
+ desired_backend="$(readlink -f "${DOCKER_IPTABLES_BACKEND}")"
+ local desired_ip6_backend
+ desired_ip6_backend="$(readlink -f "${DOCKER_IP6TABLES_BACKEND}")"
+ local current_backend
+ current_backend="$(readlink -f /etc/alternatives/iptables 2>/dev/null || true)"
+
+ test -x "${DOCKER_IPTABLES_BACKEND}"
+ test -x "${DOCKER_IP6TABLES_BACKEND}"
+
+ # The cloud image can preserve a legacy FORWARD DROP policy while apt selects
+ # iptables-nft. Make Docker program the enforcing legacy table instead of
+ # opening FORWARD or bypassing Docker's per-network isolation chains.
+ if [ "${current_backend}" != "${desired_backend}" ]; then
+ sudo /usr/bin/update-alternatives \
+ --set iptables "${DOCKER_IPTABLES_BACKEND}"
+ fi
+ if [ "$(readlink -f /etc/alternatives/ip6tables 2>/dev/null || true)" \
+ != "${desired_ip6_backend}" ]; then
+ sudo /usr/bin/update-alternatives \
+ --set ip6tables "${DOCKER_IP6TABLES_BACKEND}"
+ fi
+
+ current_backend="$(readlink -f /etc/alternatives/iptables)"
+ local current_ip6_backend
+ current_ip6_backend="$(readlink -f /etc/alternatives/ip6tables)"
+ test "${current_backend}" = "${desired_backend}"
+ test "${current_ip6_backend}" = "${desired_ip6_backend}"
+}
+
+ensure_dockerd() {
+ ensure_docker_iptables_backend
+ local storage_driver='vfs'
+ local storage_args=()
+ if [ -x /usr/bin/fuse-overlayfs ]; then
+ storage_driver='fuse-overlayfs'
+ fi
+ if ! sudo /usr/bin/grep -q '"storage-driver"[[:space:]]*:' /etc/docker/daemon.json 2>/dev/null; then
+ storage_args+=("--storage-driver=${storage_driver}")
+ fi
+ if [ -S "${DOCKER_SOCK}" ]; then
+ sudo chmod 666 "${DOCKER_SOCK}"
+ if docker info >/dev/null 2>&1; then
+ return 0
+ fi
+ fi
+ # As with Nix, a snapshot can contain a dead Unix socket. Docker does not
+ # reliably replace it while starting, so clear it after the API probe fails.
+ sudo rm -f "${DOCKER_SOCK}"
+ : >"${LOG_DIR}/dockerd.log"
+ sudo setsid /usr/bin/dockerd "${storage_args[@]}" \
+ >>"${LOG_DIR}/dockerd.log" 2>&1 </dev/null &
+ local n=0
+ while [ "${n}" -lt 60 ]; do
+ if [ -S "${DOCKER_SOCK}" ]; then
+ sudo chmod 666 "${DOCKER_SOCK}"
+ if docker info >/dev/null 2>&1; then
+ return 0
+ fi
+ fi
+ n=$((n + 1))
+ sleep 1
+ done
+ echo "dockerd did not become ready" >&2
+ sed -n '1,160p' "${LOG_DIR}/dockerd.log" >&2 || true
+ return 1
+}
+
+in_pinned_nix_shell() {
+ [ "${MACRO_CLOUD_PINNED_NIX:-}" = "1" ]
+}
+
+reenter_pinned_nix_shell() {
+ local script_path="$1"
+ shift
+ exec "${NIX_BIN}" develop "${WORKSPACE_ROOT}" --command /usr/bin/env \
+ MACRO_CLOUD_PINNED_NIX=1 /usr/bin/bash "${script_path}" "$@"
+}
+
+workspace_owner() {
+ stat -c '%U' "${WORKSPACE_ROOT}"
+}
+
+workspace_group() {
+ stat -c '%G' "${WORKSPACE_ROOT}"
+}
+
+remove_workspace_target_mounts() {
+ command -v docker >/dev/null 2>&1 || return 0
+ docker info >/dev/null 2>&1 || return 0
+ local id source
+ for id in $(docker ps -aq); do
+ while IFS= read -r source; do
+ case "${source}" in
+ "${WORKSPACE_ROOT}/target" | "${WORKSPACE_ROOT}/target"/*)
+ echo "cursor-cloud: removing leftover container ${id} mounted on ${source}"
+ docker rm -f "${id}" >/dev/null 2>&1 || true
+ break
+ ;;
+ esac
+ done < <(docker inspect --format '{{range .Mounts}}{{.Source}}{{println}}{{end}}' "${id}")
+ done
+}
+
+# Move /workspace/target into $HOME (same filesystem: rename). Recreate the
+# symlink after every checkout so Cargo's incremental cache survives it.
+ensure_persistent_caches() {
+ mkdir -p "${TARGET_CACHE}" "${FRONTEND_CACHE}" "${MACRO_STACK_SNAPSHOT_DIR}"
+ remove_workspace_target_mounts
+
+ if [ -e "${WORKSPACE_ROOT}/target" ] && [ ! -L "${WORKSPACE_ROOT}/target" ]; then
+ sudo chown -R "$(workspace_owner):$(workspace_group)" "${WORKSPACE_ROOT}/target"
+ if [ -x "${WORKSPACE_ROOT}/target/x86_64-unknown-linux-gnu/debug/document_storage_service" ] \
+ && [ ! -x "${TARGET_CACHE}/x86_64-unknown-linux-gnu/debug/document_storage_service" ]; then
+ echo "cursor-cloud: moving ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"
+ rm -rf "${TARGET_CACHE}"
+ mv "${WORKSPACE_ROOT}/target" "${TARGET_CACHE}"
+ else
+ echo "cursor-cloud: replacing ${WORKSPACE_ROOT}/target with cache symlink"
+ rm -rf "${WORKSPACE_ROOT}/target"
+ fi
+ fi
+
+ mkdir -p "${TARGET_CACHE}"
+ ln -sfn "${TARGET_CACHE}" "${WORKSPACE_ROOT}/target"
+ sudo chown -R "$(workspace_owner):$(workspace_group)" "${TARGET_CACHE}"
+ echo "cursor-cloud: ${WORKSPACE_ROOT}/target -> ${TARGET_CACHE}"
+
+ if [ ! -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ] && [ -f "${FRONTEND_CACHE}/index.html" ]; then
+ mkdir -p "${WORKSPACE_ROOT}/apps/web/dist"
+ cp -a "${FRONTEND_CACHE}/." "${WORKSPACE_ROOT}/apps/web/dist/"
+ echo "cursor-cloud: restored frontend bundle from cache"
+ elif [ -f "${WORKSPACE_ROOT}/apps/web/dist/index.html" ]; then
+ cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"
+ fi
+
+ if [ -d "${WORKSPACE_ROOT}/infra/local/generated/.snapshots" ]; then
+ cp -a "${WORKSPACE_ROOT}/infra/local/generated/.snapshots/." "${MACRO_STACK_SNAPSHOT_DIR}/"
+ fi
+}
+
+bake_stack_snapshot_image() {
+ local key="$1"
+ local snapshot_dir="$2"
+
+ test -f "${snapshot_dir}/manifest.json"
+ docker build \
+ --tag "${STACK_SNAPSHOT_IMAGE}" \
+ --file - \
+ "${snapshot_dir}" <<EOF
+FROM alpine:3
+LABEL com.macro.stack-snapshot-key="${key}"
+COPY . /snapshot/
+EOF
+ docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null
+}
+
+restore_stack_snapshot_image() {
+ if ! docker image inspect "${STACK_SNAPSHOT_IMAGE}" >/dev/null 2>&1; then
+ echo "cursor-cloud: prepared stack snapshot image not found"
+ return 0
+ fi
+
+ local key
+ key="$(docker image inspect \
+ --format '{{ index .Config.Labels "com.macro.stack-snapshot-key" }}' \
+ "${STACK_SNAPSHOT_IMAGE}")"
+ case "${key}" in
+ ''|*[!0-9a-f]*)
+ echo "prepared stack snapshot image has invalid key: ${key}" >&2
+ return 1
+ ;;
+ esac
+
+ local destination="${MACRO_STACK_SNAPSHOT_DIR}/${key}"
+ mkdir -p "${destination}"
+ docker run --rm \
+ --user "$(id -u):$(id -g)" \
+ --volume "${destination}:/restore" \
+ "${STACK_SNAPSHOT_IMAGE}" \
+ sh -ceu 'cp -R /snapshot/. /restore/'
+ test -f "${destination}/manifest.json"
+ echo "cursor-cloud: restored stack snapshot ${key} from image"
+}
diff --git a/.cursor/environment.json b/.cursor/environment.json
new file mode 100644
index 0000000000..632ad42a26
--- /dev/null
+++ b/.cursor/environment.json
@@ -0,0 +1,5 @@
+{
+ "name": "macro",
+ "install": "bash .cursor/install.sh",
+ "start": "bash .cursor/start.sh"
+}
diff --git a/.cursor/install.sh b/.cursor/install.sh
new file mode 100755
index 0000000000..4e3ad3c359
--- /dev/null
+++ b/.cursor/install.sh
@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck source=cloud-lib.sh
+source "${SCRIPT_DIR}/cloud-lib.sh"
+
+ensure_apt_packages() {
+ local pkg
+ local missing=()
+ for pkg in fuse-overlayfs libssl-dev pkg-config postgresql-client; do
+ if ! dpkg -s "${pkg}" >/dev/null 2>&1; then
+ missing+=("${pkg}")
+ fi
+ done
+ # Do not replace Docker CE on bases that already provide a complete engine.
+ if [ ! -x /usr/bin/docker ] || [ ! -x /usr/bin/dockerd ]; then
+ missing+=(docker.io docker-buildx docker-compose-v2)
+ fi
+ if [ "${#missing[@]}" -ne 0 ]; then
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq
+ sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ -o Dpkg::Options::=--force-confdef \
+ -o Dpkg::Options::=--force-confold \
+ "${missing[@]}"
+ fi
+ test -x /usr/bin/dockerd
+ /usr/bin/docker buildx version
+ /usr/bin/docker compose version
+}
+
+crate_env_paths() {
+ awk '
+ $0 == "setup_test_envs:" { p = 1; next }
+ p && /^[A-Za-z_]/ { exit }
+ p && />>/ {
+ sub(/^.*>>[[:space:]]*/, "")
+ print
+ }
+ ' /workspace/tooling/just/rust.just
+}
+
+# setup_test_envs always appends; write each DATABASE_URL at most once.
+ensure_test_envs() {
+ local f
+ while IFS= read -r f; do
+ [ -z "${f}" ] && continue
+ mkdir -p "$(dirname "${f}")"
+ if [ -f "${f}" ] && grep -qxF "DATABASE_URL=${MACRODB_URL}" "${f}"; then
+ continue
+ fi
+ printf 'DATABASE_URL=%s\n' "${MACRODB_URL}" >>"${f}"
+ done < <(crate_env_paths)
+}
+
+build_frontend_artifact() {
+ bun install --frozen-lockfile
+ (
+ cd "${WORKSPACE_ROOT}/apps/web"
+ MODE=development NODE_ENV=production VITE_LOCAL_SERVERS=ALL VITE_LOCAL_BACKEND_ORIGIN=same-origin \
+ bun run --bun build
+ )
+ test -f "${WORKSPACE_ROOT}/apps/web/dist/index.html"
+ mkdir -p "${FRONTEND_CACHE}"
+ cp -a "${WORKSPACE_ROOT}/apps/web/dist/." "${FRONTEND_CACHE}/"
+}
+
+cleanup_stack() {
+ cd "${WORKSPACE_ROOT}"
+ timeout --kill-after=10s 2m just --quiet stack down
+}
+
+cleanup_stack_best_effort() {
+ cleanup_stack >/dev/null 2>&1 || true
+}
+
+prepare_durable_stack() {
+ # `infra-only` is the existing finite CI bake mode: it materializes every
+ # Compose image, initializes the real infra, and saves/restores the
+ # content-addressed volume snapshot without starting secret-dependent apps.
+ just stack up \
+ --infra-only \
+ --no-doppler \
+ --build-aux-services \
+ --json
+
+ local snapshot_json
+ snapshot_json="$(
+ cargo run --quiet --manifest-path Cargo.toml -p xtask_local --features local-stack -- \
+ stack snapshot --json
+ )"
+ local snapshot_fields=()
+ mapfile -t snapshot_fields < <(python3 -c '
+import json
+import sys
+
+snapshot = json.load(sys.stdin)
+key = snapshot["key"]
+if not snapshot["present"]:
+ raise SystemExit(f"stack snapshot missing for key {key}")
+print(key)
+print(snapshot["dir"])
+' <<<"${snapshot_json}")
+ local snapshot_key="${snapshot_fields[0]}"
+ local snapshot_dir="${snapshot_fields[1]}"
+
+ bake_stack_snapshot_image "${snapshot_key}" "${snapshot_dir}"
+ local image_id
+ image_id="$(docker image inspect --format '{{.Id}}' "${STACK_SNAPSHOT_IMAGE}")"
+
+ echo "cursor-cloud install: stack snapshot ${snapshot_key} at ${snapshot_dir}"
+ echo "cursor-cloud install: stack snapshot image ${STACK_SNAPSHOT_IMAGE} (${image_id})"
+}
+
+if ! in_pinned_nix_shell; then
+ ensure_apt_packages
+ ensure_nix_daemon
+ ensure_dockerd
+ reenter_pinned_nix_shell "${SCRIPT_DIR}/install.sh" "$@"
+fi
+
+ensure_persistent_caches
+
+docker pull pgvector/pgvector:pg18
+docker pull redis/redis-stack:latest
+
+cd "${WORKSPACE_ROOT}"
+just run_dbs -d
+trap cleanup_stack_best_effort EXIT
+
+ensure_test_envs
+
+just initialize_dbs
+cargo fetch --locked
+unset SQLX_OFFLINE
+cargo test --no-run -p macro_db_client
+
+echo "cursor-cloud install: test-ready"
+
+# Build tools own cache validity: Bun, Cargo, and BuildKit compare the current
+# source/lock/toolchain inputs instead of trusting surviving files or image tags.
+build_frontend_artifact
+echo "cursor-cloud install: frontend artifact"
+
+# The bake primitive builds the runtime image and complete Rust binary inventory
+# before initializing and snapshotting infra.
+prepare_durable_stack
+cleanup_stack
+trap - EXIT
+
+echo "cursor-cloud install: durable stack ready"
diff --git a/.cursor/stack.sh b/.cursor/stack.sh
new file mode 100755
index 0000000000..871d4fdde0
--- /dev/null
+++ b/.cursor/stack.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# On-demand product stack. start.sh is infra-only so cargo-test agents do not
+# boot FusionAuth. Cargo, Bun, and BuildKit incrementally reconcile the checked
+# out branch against the durable main-build caches before services start.
+
+# shellcheck source=cloud-lib.sh
+source "${SCRIPT_DIR}/cloud-lib.sh"
+
+if ! in_pinned_nix_shell; then
+ ensure_nix_daemon
+ ensure_dockerd
+ reenter_pinned_nix_shell "${SCRIPT_DIR}/stack.sh" "$@"
+fi
+
+/usr/bin/bash "${SCRIPT_DIR}/start.sh"
+
+cd "${WORKSPACE_ROOT}"
+restore_stack_snapshot_image
+just stack up --no-doppler --build-aux-services
+
+echo "cursor-cloud stack: app ready"
diff --git a/.cursor/start.sh b/.cursor/start.sh
new file mode 100755
index 0000000000..2eb37885f1
--- /dev/null
+++ b/.cursor/start.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# shellcheck source=cloud-lib.sh
+source "${SCRIPT_DIR}/cloud-lib.sh"
+
+if ! in_pinned_nix_shell; then
+ ensure_nix_daemon
+ ensure_dockerd
+ reenter_pinned_nix_shell "${SCRIPT_DIR}/start.sh" "$@"
+fi
+
+ensure_persistent_caches
+
+cd "${WORKSPACE_ROOT}"
+just run_dbs -d
+
+pg_isready -h 127.0.0.1 -p 5432
+
+echo "cursor-cloud start: infra ready"
diff --git a/.dockerignore b/.dockerignore
index 581f164ed6..931cf6e3e1 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,8 +2,10 @@
.git
.jj
.agents
+.cursor
.claude/.local
.pi/outputs
+/infra/local/generated
# Legacy layout directories may still contain ignored local build artifacts.
/agents
diff --git a/CLAUDE.md b/CLAUDE.md
index d5b2b9fe79..d6baf9a84e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -78,13 +78,15 @@ just check # Type check without building
### Testing
```bash
-# Setup test environment
-docker compose --project-directory . -f docker/docker-compose.yml up -d postgres
-just setup_test_envs # Setup .env files for tests
-just initialize_dbs # Initialize all databases
-just test # Run tests
+just create_networks
+just run_dbs -d
+just setup_test_envs
+just initialize_dbs
+cargo test -p {crate}
```
+`just test` does not exist. Leave `SQLX_OFFLINE` unset when you run `cargo test`. Run `just prepare_db` only if you changed SQL queries.
+
### Pre Commit
```bash
cargo fmt # format
@@ -93,12 +95,11 @@ just clippy # extra lints / best practices
### Database Management
-```bash
-just setup_macrodb # Setup main database
-just setup_commsdb # Setup communications database
-just setup_emaildb # Setup email database
-just setup_contactsdb # Setup contacts database
-```
+Use `just setup_macrodb` or `just initialize_dbs` to create and migrate MacroDB. Those recipes are the same.
+
+Schemas live in `crates/macro_db_client/migrations/`.
+
+To reset MacroDB, run `just crates/macro_db_client/drop_db -y -f`, then `just setup_macrodb`.
### Lambda Building
@@ -316,35 +317,12 @@ don't inject it directly into the error message.
## Cursor Cloud specific instructions
-This VM runs the intended local dev stack from `docs/RUNNING_LOCALLY.md` via Nix + Docker.
-The startup layer (nix daemon, docker daemon, `bun install`) is handled by the environment
-update script; the notes below are durable, non-obvious caveats for running the app here.
-
-- Toolchain lives in the Nix dev shell. Run dev commands from the repo root as
- `nix develop --command bash -c '<cmd>'`. If `nix` is not found, first
- `. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh`. There is no systemd
- (init is `tini`); `nix-daemon` and `dockerd` are plain background processes.
-- Use the `--no-doppler` path (this repo is run without team secrets). Prefer the
- headless orchestrator `just stack ...` over `just run_local` — `run_local` has an
- interactive hotkey loop that does not suit a non-interactive agent.
-- KEY GOTCHA (openssh): `just stack up` / `run_local` shell out to `ssh-keygen`/`ssh`
- for the SDK webhook relay. The VM's `/exec-daemon/ssh-keygen` shim is first on PATH and
- crashes under the Nix shell's `LD_LIBRARY_PATH` (glibc 2.42 vs system 2.39). Fix by
- putting a self-consistent Nix openssh first on PATH. openssh is installed in the Nix
- profile (`nix profile add nixpkgs#openssh`), so launch the stack as:
- `nix develop --command bash -c 'export PATH=$HOME/.nix-profile/bin:$PATH; just stack up --no-doppler'`
-- First `stack up` cross-compiles all service binaries with `cargo-zigbuild`
- (slow — ~10 min cold on 4 cores); later runs reuse `sccache` and a cached init snapshot,
- so they are fast. `just stack status` / `just stack update` / `just stack down` manage it.
-- Endpoints (default instance): app at `http://localhost:8090/app/`, proxy at `:8090`,
- Mailpit at `http://localhost:8090/mailpit/`, FusionAuth at `:9011`.
-- Login is passwordless: the one-time code is delivered to Mailpit (not a real inbox).
-- Seed sample data: `just seed-scenario apply --file seed/scenarios/team-perms.json`
- (personas `alice`..`eve` at `<name>@seed.macro.local`). The login links it prints embed
- port `3000` (the `run_local` dev-server default); in headless `stack` mode use `:8090`,
- e.g. `http://localhost:8090/app/[email protected]`.
-- `agent_harness_service` restarts continuously in a `--no-doppler` stack (it needs AI
- provider keys); this is expected and does not affect non-AI features.
-- Tests run against the live local Postgres — do not set `SQLX_OFFLINE=true` for
- `cargo test` (see the sqlx notes above). For DB-backed crate tests, run
- `just setup_test_envs` first; pure-logic crates (e.g. `cargo test -p email_utils`) need no setup.
+Environment scripts are `.cursor/install.sh` (durable files), `.cursor/start.sh` (dockerd, then Postgres and Redis), and `.cursor/stack.sh` (on-demand product stack).
+
+After boot, Postgres should already be up. If `:5432` is closed, run `bash .cursor/start.sh`.
+
+Cargo `target/`, the frontend bundle, and the stack init snapshot live under `$HOME/.cache/macro-cloud` and are symlinked or copied back into `/workspace`. A git checkout wipes `/workspace/target`. Cargo and BuildKit decide whether the next bake is incremental.
+
+Infra-ready is enough for `cargo test -p`. Product-ready is `bash .cursor/stack.sh`, which runs `just stack up --no-doppler --build-aux-services`. Install always bakes with `--infra-only --build-aux-services`. The pinned nix shell puts OpenSSH on PATH so `ssh-keygen` is not `/exec-daemon/ssh-keygen`.
+
+Do not follow the old `docker compose -f docker/docker-compose.yml up -d postgres` line. It skips `create_networks` and Redis.
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 9fee5b771a..a696bd4290 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -315,13 +315,14 @@ services:
# Agent session trigger events -> sandboxed coding agent sessions, plus the
# session control routes, which have to be served from the process that owns
# the live transports they act on.
- # Starts credential-less (and warns); arm it with
+ # Starts when both credentials are supplied:
# `DAYTONA_API_KEY=... GITHUB_TOKEN=... just run_local`.
# ============================================================================
agent_harness_service:
<<: [*common-env, *rust-services-image]
command: ["/app/out/agent_harness_service"]
restart: on-failure
+ profiles: ["agent-harness"]
# No host port: everything reaches this through the proxy, and a second
# local instance would collide on a fixed publish.
environment:
@@ -458,7 +459,13 @@ services:
- sync-service
databases:
healthcheck:
- test: ["CMD", "curl", "-f", "http://localhost:8787/health"]
+ test:
+ [
+ "CMD",
+ "node",
+ "-e",
+ "fetch('http://localhost:8787/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
+ ]
interval: 30s
timeout: 10s
retries: 3
diff --git a/docker/lexical-service.Dockerfile.dockerignore b/docker/lexical-service.Dockerfile.dockerignore
index a431e4ddcf..5655c42d9c 100644
--- a/docker/lexical-service.Dockerfile.dockerignore
+++ b/docker/lexical-service.Dockerfile.dockerignore
@@ -2,8 +2,10 @@
.git
.jj
.agents
+.cursor
.claude/.local
.pi/outputs
+/infra/local/generated
# Legacy layout directories may still contain ignored local build artifacts.
/agents
diff --git a/docker/sync-service.Dockerfile b/docker/sync-service.Dockerfile
index b010e5be85..254d93bea9 100644
--- a/docker/sync-service.Dockerfile
+++ b/docker/sync-service.Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.7
+
# Builder: compiles the worker (Rust -> wasm + JS shim). Nothing from this
# stage's toolchain (rustc, clang, npm dev deps, the cargo target dir) is
# needed at runtime, so it all stays here.
@@ -41,19 +43,18 @@ RUN cd services/sync-service && npm ci
# Copy bebop schema and generate typescript bindings
RUN cd services/sync-service/bebop && npx bebopc build
-# Build the actual application
-RUN cd services/sync-service && worker-build --profile sync-service-release
+# Build the actual application. The workspace context changes whenever a
+# dependent Rust crate changes; preserve Cargo's target cache across those
+# BuildKit layer invalidations so branch reconciliation stays incremental.
+RUN --mount=type=cache,id=macro-sync-cargo-registry,target=/usr/local/cargo/registry \
+ --mount=type=cache,id=macro-sync-cargo-target,target=/app/target \
+ cd services/sync-service && worker-build --profile sync-service-release
# Runtime: wrangler dev serving the prebuilt worker. Needs node + the
# production npm deps (wrangler brings miniflare/workerd along), the built
# worker, the wrangler config, and the D1 migrations — nothing else.
FROM node:22-bookworm-slim
-# curl for the compose healthcheck; ca-certificates for wrangler's TLS.
-RUN apt-get update && \
- apt-get install -y --no-install-recommends curl ca-certificates && \
- rm -rf /var/lib/apt/lists/*
-
WORKDIR /app
COPY services/sync-service/package.json services/sync-service/package-lock.json ./
@@ -67,4 +68,4 @@ EXPOSE 8787
# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,
# apply local D1 migrations, then start.
-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\nDSS_INTERNAL_AUTH_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" "$DSS_INTERNAL_AUTH_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'
+CMD ["sh", "-c", "printf \"LOCAL_API_KEY=%s\\nDOCUMENT_PERMISSIONS_SECRET=%s\\nSPS_API_SECRET_KEY=%s\\nDSS_INTERNAL_AUTH_KEY=%s\\n\" \"$INTERNAL_API_SECRET_KEY\" \"$DOCUMENT_PERMISSIONS_SECRET\" \"$INTERNAL_API_SECRET_KEY\" \"$DSS_INTERNAL_AUTH_KEY\" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml"]
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index fe80c8ce04..e493a94675 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -769,6 +769,7 @@
parallel
docker-compose
curl
+ openssh
wget
kcat
xz
diff --git a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
index 57ac786768..9cf80e2279 100644
--- a/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/build/runtime_image.rs
@@ -17,25 +17,26 @@ pub const RUNTIME_IMAGE_TAG: &str = "macro-local-runtime:dev";
const DOCKERFILE_REL: &str = "docker/Dockerfile.runtime";
-/// Build the runtime image for the host arch if it is missing (or `force`).
+/// Reconcile the runtime image for the host arch through BuildKit.
+///
+/// BuildKit's content cache makes an unchanged build cheap while still
+/// invalidating the image when the Dockerfile or build context changes.
+/// Existence alone is not a valid cache key because the tag survives branch
+/// checkouts. `force` disables BuildKit's layer cache.
pub fn ensure_runtime_image(stage: &Stage, target: Target, force: bool) -> Result<()> {
- if !force && image_exists() {
- return Ok(());
- }
let ws = workspace_root();
let mut cmd = Command::new("docker");
cmd.current_dir(&ws)
.args(["buildx", "build", "--platform", target.docker_platform])
.args(["-f", DOCKERFILE_REL])
.args(["-t", RUNTIME_IMAGE_TAG])
- .arg("--load")
- .arg(".");
+ .arg("--load");
+ if force {
+ cmd.arg("--no-cache");
+ }
+ cmd.arg(".");
stage.run(
&format!("Building runtime image {RUNTIME_IMAGE_TAG}"),
&mut cmd,
)
}
-
-fn image_exists() -> bool {
- super::super::docker::image_exists(RUNTIME_IMAGE_TAG)
-}
diff --git a/tooling/xtask/crates/xtask_local/src/local/cli.rs b/tooling/xtask/crates/xtask_local/src/local/cli.rs
index 4d207609e8..168bd88b32 100644
--- a/tooling/xtask/crates/xtask_local/src/local/cli.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/cli.rs
@@ -96,7 +96,7 @@ pub struct BuildArgs {
/// Skip building; reuse whatever is in the target dir.
#[arg(long)]
pub no_build: bool,
- /// Rebuild Docker-built auxiliary services (sync, websocket, lexical).
+ /// Rebuild every repository-built local Docker service.
#[arg(long)]
pub build_aux_services: bool,
/// Use this dir as the `/app/out` source instead of building.
@@ -175,7 +175,7 @@ pub struct ValidateEnvArgs {
#[derive(Args, Clone, Default)]
pub struct ForceArg {
- /// Rebuild even if the image already exists.
+ /// Rebuild without BuildKit's layer cache.
#[arg(long)]
pub force: bool,
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/docker.rs b/tooling/xtask/crates/xtask_local/src/local/docker.rs
index 4baaefc0ab..895a13cb81 100644
--- a/tooling/xtask/crates/xtask_local/src/local/docker.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/docker.rs
@@ -1,8 +1,7 @@
//! Thin bollard-backed helpers for the discrete Docker Engine operations the
-//! orchestrator needs outside of `docker compose` — checking whether the
-//! runtime image exists and idempotently ensuring the per-instance external
-//! networks/volumes. (Compose orchestration itself stays on the CLI; bollard
-//! has no compose support.)
+//! orchestrator needs outside of `docker compose` — idempotently ensuring the
+//! per-instance external networks/volumes. (Compose orchestration itself stays
+//! on the CLI; bollard has no compose support.)
use anyhow::{Context, Result};
use bollard::Docker;
@@ -32,15 +31,6 @@ fn connect() -> Result<Docker> {
}
}
-/// Whether an image with `tag` exists locally.
-pub fn image_exists(tag: &str) -> bool {
- block_on(async {
- let docker = connect()?;
- Ok::<_, anyhow::Error>(docker.inspect_image(tag).await.is_ok())
- })
- .unwrap_or(false)
-}
-
/// Idempotently create a bridge network (no-op if it already exists).
pub fn ensure_network(name: &str) -> Result<()> {
block_on(async {
diff --git a/tooling/xtask/crates/xtask_local/src/local/mod.rs b/tooling/xtask/crates/xtask_local/src/local/mod.rs
index cdbcfd1dcf..681dce0317 100644
--- a/tooling/xtask/crates/xtask_local/src/local/mod.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/mod.rs
@@ -11,6 +11,7 @@
//! Design invariant: Docker never compiles Rust. Binaries are built on the host
//! and bind-mounted read-only into the runtime image at `/app/out`.
+use std::collections::BTreeMap;
use std::path::PathBuf;
pub mod arch;
@@ -157,7 +158,36 @@ use anyhow::Result;
use instance::Instance;
use stage::Stage;
-const AUX_SERVICE_IMAGES: &[&str] = &["websocket_service", "sync_service", "lexical_service"];
+/// Every non-Rust local service whose image is built from this repository.
+/// `docker compose up` builds a missing image implicitly, but an Environment
+/// Build must materialize all of them so a fresh agent never discovers one at
+/// stack-start time.
+const LOCAL_BUILD_SERVICE_IMAGES: &[&str] = &[
+ "websocket_service",
+ "sync_service",
+ "lexical_service",
+ "ai_editing_worker",
+ "analytics_proxy",
+ "sdk-webhook-relay",
+ "search",
+];
+
+/// Repository-built app containers safe to recreate during `stack update`.
+/// OpenSearch is built for cold-stack correctness but remains under the infra
+/// lifecycle, which waits for health before Rust services can reconnect.
+const LOCAL_RECREATE_SERVICE_IMAGES: &[&str] = &[
+ "websocket_service",
+ "sync_service",
+ "lexical_service",
+ "ai_editing_worker",
+ "analytics_proxy",
+ "sdk-webhook-relay",
+];
+
+/// Image-only app services that infra-only bake mode does not otherwise start.
+/// Infra images are pulled by `bring_up_infra`; the Rust runtime image is built
+/// separately by [`build::ensure_runtime_image`].
+const LOCAL_PULL_SERVICE_IMAGES: &[&str] = &["proxy", "mailpit", "static_file_cdn"];
/// The artifact-driven preview updater can refresh every Docker-built app
/// service. Keep this separate so enabling local aux rebuilds does not make the
@@ -210,7 +240,7 @@ pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
// Foreground: resolve env, build binaries + runtime image, generate the
// compose override / Caddyfile / kickstart. None of this touches the volumes
// or containers the teardown is removing, so it's safe to overlap.
- let (env, target) = prepare(&stage, mode, &instance, args, false)?;
+ let (env, target) = prepare(&stage, mode, &instance, args, false, false)?;
// Join the background teardown before we (re)create volumes + bring infra up,
// surfaced as a live spinner so it's clear what we're blocked on. It
@@ -474,6 +504,7 @@ fn prepare(
instance: &Instance,
args: &cli::RunArgs,
static_frontend: bool,
+ pull_app_images: bool,
) -> Result<(env_layer::ResolvedEnv, arch::Target)> {
let env = env_layer::resolve(
mode,
@@ -516,6 +547,9 @@ fn prepare(
if args.build.build_aux_services {
build_aux_service_images(stage, instance, &env)?;
}
+ if pull_app_images {
+ pull_app_service_images(stage, instance, &env)?;
+ }
Ok((env, target))
}
@@ -523,7 +557,17 @@ fn prepare(
/// generated env. Shared by bring-up and the binary-reload restart.
fn compose_cmd(instance: &Instance, env: &env_layer::ResolvedEnv) -> Command {
let files = gen_compose::compose_files(instance);
- gen_compose::docker_compose(instance, &files, &env.generated_path)
+ let mut command = gen_compose::docker_compose(instance, &files, &env.generated_path);
+ if agent_harness_enabled(&env.merged) {
+ command.args(["--profile", "agent-harness"]);
+ }
+ command
+}
+
+fn agent_harness_enabled(env: &BTreeMap<String, String>) -> bool {
+ ["DAYTONA_API_KEY", "GITHUB_TOKEN"]
+ .iter()
+ .all(|key| env.get(*key).is_some_and(|value| !value.trim().is_empty()))
}
fn build_aux_service_images(
@@ -532,10 +576,20 @@ fn build_aux_service_images(
env: &env_layer::ResolvedEnv,
) -> Result<()> {
let mut build = compose_cmd(instance, env);
- build.arg("build").args(AUX_SERVICE_IMAGES);
+ build.arg("build").args(LOCAL_BUILD_SERVICE_IMAGES);
stage.run("Building auxiliary service images", &mut build)
}
+fn pull_app_service_images(
+ stage: &Stage,
+ instance: &Instance,
+ env: &env_layer::ResolvedEnv,
+) -> Result<()> {
+ let mut pull = compose_cmd(instance, env);
+ pull.arg("pull").args(LOCAL_PULL_SERVICE_IMAGES);
+ stage.run("Pulling image-only app services", &mut pull)
+}
+
fn recreate_aux_service_containers(
stage: &Stage,
instance: &Instance,
@@ -543,7 +597,7 @@ fn recreate_aux_service_containers(
) -> Result<()> {
let mut up = compose_cmd(instance, env);
up.args(["up", "-d", "--force-recreate", "--no-deps"])
- .args(AUX_SERVICE_IMAGES);
+ .args(LOCAL_RECREATE_SERVICE_IMAGES);
stage.run("Recreating auxiliary service containers", &mut up)
}
diff --git a/tooling/xtask/crates/xtask_local/src/local/stack.rs b/tooling/xtask/crates/xtask_local/src/local/stack.rs
index 175e9e6a62..f6dc0f4aa4 100644
--- a/tooling/xtask/crates/xtask_local/src/local/stack.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/stack.rs
@@ -73,7 +73,7 @@ pub struct UpdateArgs {
/// Apply this complete prebuilt binary set instead of invoking Cargo.
#[arg(long, conflicts_with = "build_aux_services")]
pub binaries_dir: Option<PathBuf>,
- /// Rebuild the Docker-built auxiliary services (sync, websocket, lexical).
+ /// Rebuild every repository-built local Docker service.
#[arg(long)]
pub build_aux_services: bool,
/// Recreate registry-refreshed preview auxiliary services without building.
@@ -180,7 +180,14 @@ pub fn up(mode: Mode, args: &UpArgs) -> Result<Instance> {
});
let static_frontend = !args.run.no_frontend && !args.infra_only;
- let (env, target) = super::prepare(&stage, mode, &instance, &args.run, static_frontend)?;
+ let (env, target) = super::prepare(
+ &stage,
+ mode,
+ &instance,
+ &args.run,
+ static_frontend,
+ args.infra_only,
+ )?;
let configured_binaries = args
.run
.build
diff --git a/tooling/xtask/crates/xtask_local/src/local/test.rs b/tooling/xtask/crates/xtask_local/src/local/test.rs
index 99d6812947..4bfe836257 100644
--- a/tooling/xtask/crates/xtask_local/src/local/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/test.rs
@@ -1,4 +1,5 @@
use super::*;
+use std::collections::{BTreeMap, BTreeSet};
/// Every mode whose spec we assert invariants over.
const MODES: &[Mode] = &[Mode::Local, Mode::Dev];
@@ -25,3 +26,60 @@ fn mode_specs_are_coherent() {
);
}
}
+
+#[test]
+fn durable_bake_covers_every_repository_built_local_image() {
+ fn direct_build_services(path: &std::path::Path) -> BTreeSet<String> {
+ let raw = std::fs::read_to_string(path).unwrap();
+ let compose: serde_yaml::Value = serde_yaml::from_str(&raw).unwrap();
+ compose["services"]
+ .as_mapping()
+ .unwrap()
+ .iter()
+ .filter_map(|(name, service)| {
+ service
+ .as_mapping()
+ .is_some_and(|service| service.contains_key("build"))
+ .then(|| name.as_str().unwrap().to_string())
+ })
+ .collect()
+ }
+
+ let mut expected = direct_build_services(&repo_root().join("docker/docker-compose.yml"));
+ expected.extend(direct_build_services(
+ &repo_root().join("docker/docker-compose-databases.yml"),
+ ));
+ // The generated override adds this build definition.
+ expected.insert("sdk-webhook-relay".to_string());
+ // The Rust services use the host-built runtime image; this profile-only
+ // helper is intentionally never needed by local stack preparation.
+ expected.remove("rust_services_image");
+ for service in inventory::RUST_SERVICES {
+ expected.remove(service.compose_name);
+ }
+
+ let actual: BTreeSet<_> = LOCAL_BUILD_SERVICE_IMAGES
+ .iter()
+ .map(|name| name.to_string())
+ .collect();
+ assert_eq!(actual, expected);
+ assert_eq!(
+ LOCAL_PULL_SERVICE_IMAGES,
+ ["proxy", "mailpit", "static_file_cdn"]
+ );
+}
+
+#[test]
+fn agent_harness_requires_both_credentials() {
+ let mut env = BTreeMap::new();
+ assert!(!agent_harness_enabled(&env));
+
+ env.insert("DAYTONA_API_KEY".into(), "daytona".into());
+ assert!(!agent_harness_enabled(&env));
+
+ env.insert("GITHUB_TOKEN".into(), "github".into());
+ assert!(agent_harness_enabled(&env));
+
+ env.insert("DAYTONA_API_KEY".into(), " ".into());
+ assert!(!agent_harness_enabled(&env));
+}