Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

52.1 KB
Raw Blame
//! xtask-owned local & dev orchestration.
//!
//! `just run_local`, `just run_dev`, and the supporting
//! `doctor`/`validate`/`status`/`stop`/`reset`/`destroy` recipes all delegate here.
//! This module owns: building Linux service binaries on the host (zigbuild),
//! the minimal runtime image, the typed service inventory, per-instance
//! isolation, environment layering, generated docker-compose overrides, local
//! infra (FusionAuth via kickstart, Mailpit, LocalStack, Postgres/Redis, DB
//! migrate), the single-origin reverse proxy, and the frontend dev server.
//!
//! Design invariant: Docker never compiles Rust. Binaries are built on the host
//! and bind-mounted read-only into the runtime image at `/app/out`.

use std::path::PathBuf;

pub mod arch;
pub mod build;
pub mod cf_tunnel;
pub mod cli;
pub mod db;
pub mod docker;
pub mod doctor;
pub mod e2e;
pub mod env_layer;
pub mod frontend;
pub mod fusionauth;
pub mod gen_compose;
pub mod identity;
pub mod instance;
pub mod inventory;
pub mod kafka;
pub mod kickstart;
pub mod local_env;
pub mod localstack;
pub mod mailpit;
pub mod opensearch;
pub mod portmap;
pub mod proxy;
pub mod resources;
pub mod sandbox_image;
pub mod sdk_webhook;
pub mod seed_env;
pub mod snapshot;
pub mod stack;
pub mod stage;
pub mod status;
pub mod summary;
pub mod tls;
pub mod validate;

#[cfg(test)]
mod test;

/// What flavor of stack we are bringing up. The mode drives which infra runs,
/// which env applies, and which service subset starts.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Mode {
    /// Fully local: local Postgres/Redis/FusionAuth/Mailpit/LocalStack + app
    /// services + proxy + frontend. No Pulumi, no real AWS, no SES.
    Local,
    /// Local binaries pointed at shared dev resources. No local infra.
    Dev,
}

/// The per-mode behavior table. Each [`Mode`] resolves to exactly one
/// `ModeSpec`, and the orchestration code reads these fields instead of matching
/// on the enum — so a mode's behavior lives in one place and adding a mode is one
/// new [`Mode::spec`] arm rather than a hunt through ten files.
#[derive(Clone, Copy, Debug)]
pub struct ModeSpec {
    /// Human-facing label (`local` / `dev`).
    pub label: &'static str,
    /// The Doppler `local`-project config pulled as the secret base.
    pub doppler_config: &'static str,
    /// Overlay the code-owned [`local_env::LocalEnv`] on top of Doppler (local
    /// plumbing is authoritative). Off for dev, which runs against Doppler's
    /// dev values.
    pub overlay_local_env: bool,
    /// Resolve the developer's real AWS credentials and strip any local-only
    /// endpoints — dev talks to real AWS/RDS, not LocalStack/Postgres.
    pub uses_remote_aws: bool,
    /// Run the local-only infra: the FusionAuth kickstart, Mailpit, and the
    /// per-instance Postgres/Redis/OpenSearch isolation. Off for dev, which
    /// shares the deployed resources (LocalStack + Redis still run either way).
    pub runs_local_infra: bool,
    /// Migrate the local DB after bring-up. Off for dev — `DATABASE_URL` is the
    /// shared dev RDS and must never be migrated from here.
    pub migrates_db: bool,
    /// Route `/static-file` through LocalStack S3 (vs. the dev-pointed service).
    pub static_files_via_localstack: bool,
    /// Wait for the backend (proxy `/auth/health`) before starting the frontend.
    pub wait_backend_before_frontend: bool,
    /// Show the local infra endpoints (proxy/FusionAuth/Mailpit/…) in the
    /// summary.
    pub show_infra_in_summary: bool,
}

impl Mode {
    /// The behavior table for this mode (see [`ModeSpec`]).
    pub const fn spec(self) -> ModeSpec {
        match self {
            Mode::Local => ModeSpec {
                label: "local",
                doppler_config: "lcl_personal",
                overlay_local_env: true,
                uses_remote_aws: false,
                runs_local_infra: true,
                migrates_db: true,
                static_files_via_localstack: true,
                wait_backend_before_frontend: true,
                show_infra_in_summary: true,
            },
            Mode::Dev => ModeSpec {
                label: "dev",
                doppler_config: "dev_personal",
                overlay_local_env: false,
                uses_remote_aws: true,
                runs_local_infra: false,
                migrates_db: false,
                static_files_via_localstack: false,
                wait_backend_before_frontend: false,
                show_infra_in_summary: false,
            },
        }
    }

    pub fn label(self) -> &'static str {
        self.spec().label
    }

    /// The value of the `ENVIRONMENT` variable services read (`macro_env`).
    /// Always `local`, including dev. In the `dev`/`prod` macro_env environments
    /// the `remote_env_var` layer treats each config var's env value as the NAME
    /// of a Secrets Manager secret to fetch; only `local` reads the env value as
    /// the value itself. The Doppler `dev_personal` config ships real values (a
    /// postgres URL, keys), not secret names, so `run_dev` must run as `local`
    /// and gets its "dev-ness" from those values (dev DB + real AWS), not from
    /// switching macro_env to `dev`.
    pub fn environment_var(self) -> &'static str {
        "local"
    }
}

/// The repository Cargo workspace root. Anchored on the manifest dir rather
/// than the invocation cwd so the orchestrator works from anywhere.
pub fn workspace_root() -> PathBuf {
    xtask_paths::workspace_root()
}

/// The repository root. This is where `docker/`, `infra/`, `apps/`, and the
/// root `justfile` live.
pub fn repo_root() -> PathBuf {
    xtask_paths::repo_root()
}

use std::process::Command;

use anyhow::Result;

use instance::{Instance, Port};
use stage::Stage;

/// Every non-Rust local service whose image is built from this repository.
/// `docker compose up` builds a missing image implicitly, but an Environment
/// Build must materialize all of them so a fresh agent never discovers one at
/// stack-start time.
const LOCAL_BUILD_SERVICE_IMAGES: &[&str] = &[
    "websocket_service",
    "sync_service",
    "lexical_service",
    "ai_editing_worker",
    "analytics_proxy",
    "sdk-webhook-relay",
    "search",
    "headless-chrome",
];

/// Repository-built app containers safe to recreate during `stack update`.
/// OpenSearch is built for cold-stack correctness but remains under the infra
/// lifecycle, which waits for health before Rust services can reconnect.
const LOCAL_RECREATE_SERVICE_IMAGES: &[&str] = &[
    "websocket_service",
    "sync_service",
    "lexical_service",
    "ai_editing_worker",
    "analytics_proxy",
    "sdk-webhook-relay",
];

/// Image-only app services that infra-only bake mode does not otherwise start.
/// Infra images are pulled by `bring_up_infra`; the Rust runtime image is built
/// separately by [`build::ensure_runtime_image`].
const LOCAL_PULL_SERVICE_IMAGES: &[&str] = &["proxy", "mailpit", "static_file_cdn"];

/// Bring up a Local or Dev stack and (unless `--no-frontend`) the frontend.
pub fn run_stack(mode: Mode, args: &cli::RunArgs) -> Result<()> {
    if mode.spec().runs_local_infra {
        // This mode will provision Kafka topics mid-bring-up; reject a build
        // that can't do that before any containers start.
        kafka::ensure_available(&format!("run-{}", mode.label()))?;
    }
    let stage = Stage::from_env_cli(args.verbose);
    let instance = Instance::derive(args.instance.instance.as_deref(), args.instance.port_base)?;
    stage.section(&format!(
        "macro {} stack — instance {}",
        mode.label(),
        instance.name()
    ));
    if !stage.is_dry_run() {
        stack::clear_state(&instance)?;
    }

    // Before `prepare` (which resolves env and reads the OTLP port to decide
    // whether to wire `OTEL_EXPORTER_OTLP_ENDPOINT`), so a `--traces` run gets
    // the same auto-wiring as a collector started manually beforehand.
    ensure_tracing_backend(&stage, args.traces)?;
    if args.with_chrome {
        ensure_headless_chrome(&stage);
    }

    // `run_local`/`run_dev` are full delete + full create: tear the previous
    // stack and ALL its stateful volumes down so the bring-up is always from a
    // clean slate. That makes the command unconditionally idempotent — no
    // persisted state (a stale FusionAuth DB, an old migration) can poison a
    // later run. (Unclean exits skip the quit-time teardown, so this is also the
    // idempotency safety net.) Run it in the BACKGROUND: tearing the old stack
    // down is independent of the host-side build, so the two overlap instead of
    // summing (stopping ~20 containers is otherwise the slow part of a re-run).
    let teardown = (!stage.is_dry_run()).then(|| {
        let instance = instance.clone();
        std::thread::spawn(move || teardown_commands(&instance))
    });

    // Sharing posture (`--with-cf-tunnel`, local only): besides the tunnels,
    // the proxy additionally serves the headless static bundle, because that
    // bundle — built against the `same-origin` sentinel — is the only frontend
    // a remote visitor can actually use: the dev server's bundle calls the
    // backend on an absolute `localhost:<proxy>` origin. The dev server still
    // runs for the local developer. What a visitor can't do: follow
    // backend-generated absolute links (invite/login emails point at
    // `localhost:{FRONTEND_PORT}`) or the FusionAuth OAuth flow — passwordless
    // login works, with the code readable at `<tunnel>/mailpit`.
    let share_app = args.with_cf_tunnel && mode == Mode::Local;
    if args.with_cf_tunnel && mode != Mode::Local {
        stage.note("--with-cf-tunnel is ignored by run_dev (tunnels share a fully local stack)");
    }

    // The Cursor egress tunnel, before env resolution because the minted
    // hostname overrides `AgentHarnessEgressUrl`. Best-effort with a loud
    // downgrade: a laptop with no route to Cloudflare should still get a
    // working stack, minus the one thing that needs public ingress -
    // `@cursor` sessions reaching local MCP servers.
    let egress_tunnel = (share_app && !stage.is_dry_run())
        .then(|| {
            match cf_tunnel::open(&instance, "egress", instance.port(Port::AgentHarnessEgress)) {
                Ok(tunnel) => {
                    stage.note(&format!("cursor egress tunnel: {}", tunnel.url));
                    Some(tunnel)
                }
                Err(error) => {
                    stage.note(&format!(
                        "WARNING: no cursor egress tunnel ({error:#}); the egress URL stays \
                         in-network, so @cursor sessions cannot reach this stack's MCP servers"
                    ));
                    None
                }
            }
        })
        .flatten();

    // The preview SSH tunnel, also before env resolution: its hostname is what
    // the SharePreview script dials. A `tcp://` origin rather than the usual
    // `http://` — the far side is an SSH transport, not a web server — so the
    // agent reaches it through `cloudflared access ssh` as a ProxyCommand.
    // Without it a preview is still shareable, just only by an agent running on
    // this machine or in the compose network.
    let preview_ssh_tunnel = (share_app && !stage.is_dry_run())
        .then(|| {
            match cf_tunnel::open_tcp(&instance, "preview-ssh", instance.port(Port::PreviewSsh)) {
                Ok(tunnel) => {
                    stage.note(&format!("preview ssh tunnel: {}", tunnel.hostname()));
                    Some(tunnel)
                }
                Err(error) => {
                    stage.note(&format!(
                        "WARNING: no preview ssh tunnel ({error:#}); agents off this machine \
                         cannot share a live preview"
                    ));
                    None
                }
            }
        })
        .flatten();
    // Borrowed by `prepare` below, so the guard itself has to outlive it.
    let preview_ssh_tunnel_host = preview_ssh_tunnel
        .as_ref()
        .map(|tunnel| tunnel.hostname().to_owned());

    // The app tunnel does not feed the env, but it degrades the same way: a
    // failure warns and the stack comes up localhost-only.
    let app_tunnel = (share_app && !stage.is_dry_run())
        .then(
            || match cf_tunnel::open(&instance, "app", instance.port(Port::Proxy)) {
                Ok(tunnel) => {
                    stage.note(&format!("shared app tunnel: {}/app/", tunnel.url));
                    Some(tunnel)
                }
                Err(error) => {
                    stage.note(&format!(
                        "WARNING: no shared app tunnel ({error:#}); the app stays reachable \
                         on localhost only"
                    ));
                    None
                }
            },
        )
        .flatten();

    // Foreground: resolve env, build binaries + runtime image, generate the
    // compose override / Caddyfile / kickstart. None of this touches the volumes
    // or containers the teardown is removing, so it's safe to overlap.
    let (env, target) = prepare(
        &stage,
        mode,
        &instance,
        args,
        share_app,
        false,
        false,
        local_env::Tunnels {
            egress: egress_tunnel.as_ref().map(|tunnel| tunnel.url.as_str()),
            preview_ssh: preview_ssh_tunnel_host.as_deref(),
        },
    )?;

    // Build + stage the shareable bundle in the background (pure host-side
    // work), joined just before `bring_up_app` creates the proxy container
    // that mounts the staged dir. The bundle is a share-time snapshot: the
    // `r` hotkey reloads service binaries, not this bundle — live frontend
    // edits show on the dev server, not on the shared URL.
    if share_app && stage.is_dry_run() {
        frontend::build_static(&stage, &instance, mode)?;
    }
    let fe_build = (share_app && !stage.is_dry_run()).then(|| {
        let instance = instance.clone();
        let stage = stage.background();
        std::thread::spawn(move || frontend::build_static(&stage, &instance, mode))
    });

    // The init snapshot decision, exactly as `stack up` makes it (see
    // `stack::up`): hash the init-defining inputs — possible only after
    // `prepare` wrote the kickstart — and check for a stored snapshot. A hit
    // skips migrate + Kafka topics + the ~1min FusionAuth kickstart + index
    // creation; a miss runs the real init and saves one, which is how the cache
    // seeds itself. The full-delete/full-create idempotency is unchanged: the
    // key *is* the definition of clean, so any input change is a structural miss.
    let snapshot_plan = (!args.no_snapshot && !stage.is_dry_run())
        .then(|| snapshot::Plan::compute(&instance))
        .transpose()?;

    // Join the background teardown before we (re)create volumes + bring infra up,
    // surfaced as a live spinner so it's clear what we're blocked on. It
    // overlapped the build above, so this is just whatever's left of stopping the
    // old containers — instant if it already finished.
    if let Some(handle) = teardown {
        stage.run_step("Tearing down previous stack", move || {
            let _ = handle.join();
            Ok(())
        })?;
    }
    // Both modes run at least Redis + LocalStack locally, and those reference the
    // instance's `external` volumes/networks — which must exist before compose
    // `up`. Unconditional + idempotent, mirroring the unconditional teardown (dev
    // was tearing `macro_redis_data` down each run but never recreating it).
    ensure_external_resources(&stage, &instance)?;

    // Bring the backend infra up and fully ready — DB created + migrated,
    // LocalStack provisioned, FusionAuth kickstarted — BEFORE the app services
    // start. The teardown means everything is freshly created each run, so
    // otherwise the services race their backends on startup (no `macrodb`,
    // DynamoDB/OpenSearch connection refused).
    match &snapshot_plan {
        Some(plan) if plan.exists() => {
            snapshot::restore(&stage, &instance, plan)?;
            bring_up_infra(&stage, mode, &instance, &env, InfraInit::FromSnapshot)?;
        }
        Some(plan) => {
            bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?;
            snapshot::save(&stage, &instance, &env, plan)?;
        }
        None => bring_up_infra(&stage, mode, &instance, &env, InfraInit::Full)?,
    }
    if let Some(handle) = fe_build {
        stage.run_step("Building frontend (static bundle)", move || {
            handle
                .join()
                .map_err(|_| anyhow::anyhow!("frontend build panicked"))?
        })?;
    }
    bring_up_app(&stage, mode, &instance, &env)?;
    let _sdk_webhook_tunnel = (mode == Mode::Local && !stage.is_dry_run())
        .then(|| sdk_webhook::start(&instance))
        .transpose()?;

    // No restart-to-reload step: the teardown means `up` always creates fresh
    // containers, which start on the just-built binaries bind-mounted at
    // `/app/out`. (Reloading binaries into an already-running stack is the `r`
    // hotkey's job — see `restart_services`.)

    // Start the frontend (readiness-gated, output suppressed) BEFORE the
    // summary so the summary is the last thing printed; then block on the dev
    // server so `just run_local` stays attached (Ctrl-C stops it while the
    // backend containers keep running).
    let frontend = if args.no_frontend {
        stage.note("frontend disabled (--no-frontend)");
        None
    } else {
        // Mirror `env_layer::resolve`'s port probe: if the collector never
        // came up (best-effort default backend), don't point the browser
        // exporter at a dead endpoint.
        frontend::start(
            &stage,
            &instance,
            mode,
            args.traces.enabled() && summary::port_open(4318),
        )?
    };

    // Sharing moves the Mailpit UI under `/mailpit` (MP_WEBROOT, so a remote
    // visitor can read their login code through the proxy); show the URL that
    // actually resolves.
    let mailpit_url = if share_app {
        mailpit::proxy_ui_url(&instance)
    } else {
        mailpit::direct_ui_url(&instance)
    };
    let shared_app_url = app_tunnel
        .as_ref()
        .map(|tunnel| format!("{}/app/", tunnel.url));
    stage.print_timings("bring-up");
    summary::print(
        mode,
        &instance,
        &env,
        &frontend::https_url(&instance)?,
        &mailpit_url,
        shared_app_url.as_deref(),
    );

    match frontend {
        // Interactive terminal: stay attached with a hotkey loop.
        Some(mut fe) if stage.is_tty() => {
            interact(
                &stage,
                mode,
                &instance,
                &env,
                target,
                args.build.build_aux_services,
                &mut fe,
            )?;
        }
        // Non-interactive (piped/CI): just hold the dev server until it exits.
        Some(mut fe) => {
            let status = fe.process.child.wait()?;
            if !status.success() {
                let out = fe.tail_output(40);
                if !out.trim().is_empty() {
                    eprintln!("{out}");
                }
                anyhow::bail!("frontend dev server exited with {status}");
            }
        }
        None => {}
    }
    Ok(())
}

/// Print the hotkey legend shown while attached to a running stack.
fn print_hotkeys(stage: &Stage) {
    stage.note(
        "  [r] rebuild & reload services   [f] restart Vite   [q] quit (tears the stack down)",
    );
}

/// Re-build the service binaries on the host and restart only the containers
/// whose binary actually changed — the body of the `r` hotkey. The binaries are
/// bind-mounted at `/app/out`, so a rebuild updates them in-place; the container
/// just needs to re-exec. We snapshot binary mtimes around the build so a small
/// change relinks (and restarts) one or two services, not all twelve — and a
/// no-op build restarts nothing. Runs under a single parent spinner (quiet
/// sub-stage) so a reload is one resolving line.
fn rebuild_and_reload(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
    target: arch::Target,
    build_aux_services: bool,
) -> Result<()> {
    if stage.is_verbose() {
        // Run each step on the parent stage so the build (per group) and the
        // reload each show their own `Done <elapsed>` — the build-vs-reload
        // split — rather than folding into one line.
        run_rebuild(stage, mode, instance, env, target, build_aux_services)
    } else {
        stage.run_step("Rebuilding & reloading services", || {
            run_rebuild(
                &stage.quiet(),
                mode,
                instance,
                env,
                target,
                build_aux_services,
            )
        })
    }
}

/// Snapshot binary mtimes, rebuild, and restart only the services whose binary
/// the build rewrote. Runs on a quiet sub-stage when folded, or the parent stage
/// under `--verbose` so each step times itself.
fn run_rebuild(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
    target: arch::Target,
    build_aux_services: bool,
) -> Result<()> {
    let before: Vec<Option<std::time::SystemTime>> = inventory::services_for_mode(mode)
        .map(|svc| binary_mtime(target, svc))
        .collect();
    if build_aux_services {
        std::thread::scope(|scope| {
            let rust_build = scope.spawn(|| {
                build::resolve(
                    stage,
                    target,
                    &build::BuildOptions {
                        no_build: false,
                        binaries_dir: None,
                    },
                )
            });
            let aux_build = scope.spawn(|| build_aux_service_images(stage, instance, env));
            let binaries = rust_build
                .join()
                .map_err(|_| anyhow::anyhow!("Rust service build panicked"))?;
            let aux = aux_build
                .join()
                .map_err(|_| anyhow::anyhow!("auxiliary service build panicked"))?;
            binaries?;
            aux
        })?;
    } else {
        build::resolve(
            stage,
            target,
            &build::BuildOptions {
                no_build: false,
                binaries_dir: None,
            },
        )?;
    }
    let changed: Vec<&inventory::RustService> = inventory::services_for_mode(mode)
        .zip(before)
        .filter(|(svc, was)| binary_mtime(target, svc) != *was)
        .map(|(svc, _)| svc)
        .collect();
    if build_aux_services {
        recreate_aux_service_containers(stage, instance, env)?;
    }
    if !changed.is_empty() {
        reload_services(stage, instance, env, &changed)?;
    }
    Ok(())
}

/// Modification time of a service's built binary (`None` if absent). Used to tell
/// which binaries a rebuild rewrote.
fn binary_mtime(
    target: arch::Target,
    svc: &inventory::RustService,
) -> Option<std::time::SystemTime> {
    let path = workspace_root()
        .join(target.debug_dir())
        .join(svc.cargo_bin);
    std::fs::metadata(path).and_then(|m| m.modified()).ok()
}

/// Stay attached to the running stack, handling hotkeys until the user quits or
/// the frontend exits. `r` rebuilds + reloads the services; `f` restarts Vite;
/// `q`/Esc/Ctrl-C stops
/// the frontend and tears the whole stack down (so the next run starts clean and
/// fast). An unexpected frontend exit just returns, leaving the stack up for
/// inspection — the next run's start-of-run teardown will reclaim it.
fn interact(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
    target: arch::Target,
    build_aux_services: bool,
    fe: &mut frontend::Frontend,
) -> Result<()> {
    use console::Key;

    let term = console::Term::stdout();
    print_hotkeys(stage);
    loop {
        // Noticed on the next keypress; the frontend running silently is fine.
        if let Some(status) = fe.process.child.try_wait()? {
            let out = fe.tail_output(40);
            if !out.trim().is_empty() {
                stage.note("frontend output (last lines):");
                for line in out.lines() {
                    println!("  {line}");
                }
            }
            stage.note(&format!("frontend dev server exited ({status})"));
            return Ok(());
        }
        match term.read_key() {
            Ok(Key::Char('r' | 'R')) => {
                // Drop the legend so the rebuild's single resolved line slots in
                // above a freshly reprinted legend (pinned to the bottom) rather
                // than the whole block stacking on each press.
                let _ = term.clear_last_lines(1);
                // run_step renders ✗ + the captured build error on failure; keep
                // the loop alive so the user can fix and press `r` again.
                let _ = rebuild_and_reload(stage, mode, instance, env, target, build_aux_services);
                print_hotkeys(stage);
            }
            Ok(Key::Char('f' | 'F')) => {
                let _ = term.clear_last_lines(1);
                // A failed restart returns with captured output and leaves the
                // backend containers running, like an unexpected frontend exit.
                fe.restart(stage)?;
                print_hotkeys(stage);
            }
            Ok(Key::Char('q' | 'Q') | Key::Escape | Key::CtrlC) | Err(_) => {
                // Kill the frontend's whole process group (bun + Vite), not just
                // bun — otherwise Vite is orphaned and keeps holding the port.
                fe.shutdown();
                // Tear the stack down on quit, while the user is already leaving:
                // stopping the running containers now is the slow part of
                // teardown, so doing it here makes the *next* `run_local` skip it
                // (its start-of-run teardown then has nothing to stop). Best-effort
                // — we're exiting regardless.
                let _ = teardown(stage, instance);
                return Ok(());
            }
            _ => {}
        }
    }
}

/// The shared front half of every stack flow: resolve the env, ensure the
/// runtime image + service binaries, and generate the compose override /
/// Caddyfile / kickstart. Deliberately does NOT create the external
/// networks/volumes — that's done after the background teardown joins, since
/// teardown removes them. `static_frontend` wires the proxy to serve the staged
/// app bundle (headless `stack up`, and `run_local --with-cf-tunnel`'s shared
/// app). `infra_only` skips zigbuild: bake never
/// starts Rust services and runs in parallel with the cargo lane. Returns the
/// resolved env + build target.
///
/// One argument per independent knob; bundling some into a struct would only
/// move the same list one level down.
#[allow(clippy::too_many_arguments)]
fn prepare(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    args: &cli::RunArgs,
    static_frontend: bool,
    pull_app_images: bool,
    infra_only: bool,
    tunnels: local_env::Tunnels<'_>,
) -> Result<(env_layer::ResolvedEnv, arch::Target)> {
    let env = env_layer::resolve(
        mode,
        instance,
        args.env.no_doppler,
        args.env.env_file.as_deref(),
        static_frontend,
        tunnels,
        args.traces.enabled(),
    )?;
    stage.note(&format!("env: {}", env_layer::summarize(&env.merged)));
    sandbox_image::ensure(stage, &env.merged, args.build.no_build)?;

    // Build the runtime image (idempotent) and the service binaries.
    let target = arch::detect()?;
    build::ensure_runtime_image(stage, target, false)?;
    let binaries = if infra_only {
        // Compose still emits `/app/out` mounts, but infra-only never starts
        // those services. Bake runs this in parallel with zigbuild.
        build::BinariesDir::TargetDir(workspace_root().join(target.debug_dir()))
    } else {
        build::resolve(
            stage,
            target,
            &build::BuildOptions {
                no_build: args.build.no_build,
                binaries_dir: args.build.binaries_dir.clone(),
            },
        )?
    };

    // Generate the override + the Caddyfile (the frontend reaches the services
    // through the proxy in every mode), and — for the self-contained local
    // stacks — the FusionAuth kickstart. (External networks/volumes are created
    // by the caller after the background teardown joins.)
    let gmail_forwarder = env
        .merged
        .get("GMAIL_FORWARDER_SA_KEY")
        .is_some_and(|key| !key.trim().is_empty());
    gen_compose::generate(mode, instance, &binaries, static_frontend, gmail_forwarder)?;
    proxy::write_caddyfile(instance, mode, static_frontend)?;
    if mode == Mode::Local {
        portmap::write(instance)?;
    }
    if mode.spec().runs_local_infra {
        let google = kickstart::GoogleIdp::from_env(&env.merged);
        let github = kickstart::GithubIdp::from_env(&env.merged);
        fusionauth::write_kickstart(instance, google.as_ref(), github.as_ref())?;
    }
    if args.build.build_aux_services {
        build_aux_service_images(stage, instance, &env)?;
    }
    if pull_app_images {
        pull_app_service_images(stage, instance, &env)?;
    }
    Ok((env, target))
}

/// A `docker compose` command wired for this instance's full file set + the
/// generated env. Shared by bring-up and the binary-reload restart.
fn compose_cmd(instance: &Instance, env: &env_layer::ResolvedEnv) -> Command {
    let files = gen_compose::compose_files(instance);
    gen_compose::docker_compose(instance, &files, &env.generated_path)
}

fn build_aux_service_images(
    stage: &Stage,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
) -> Result<()> {
    let mut build = compose_cmd(instance, env);
    build.arg("build").args(LOCAL_BUILD_SERVICE_IMAGES);
    stage.run("Building auxiliary service images", &mut build)
}

fn pull_app_service_images(
    stage: &Stage,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
) -> Result<()> {
    let mut pull = compose_cmd(instance, env);
    pull.arg("pull").args(LOCAL_PULL_SERVICE_IMAGES);
    stage.run("Pulling image-only app services", &mut pull)
}

fn recreate_aux_service_containers(
    stage: &Stage,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
) -> Result<()> {
    let mut up = compose_cmd(instance, env);
    up.args(["up", "-d", "--force-recreate", "--no-deps"])
        .args(LOCAL_RECREATE_SERVICE_IMAGES);
    stage.run("Recreating auxiliary service containers", &mut up)
}

/// How the local infra reaches its initialized state on bring-up.
#[derive(Clone, Copy, PartialEq, Eq)]
enum InfraInit {
    /// Run the real init: migrate the DB, let FusionAuth apply its kickstart,
    /// create the OpenSearch indices.
    Full,
    /// The volumes were restored from an init snapshot — the state already
    /// exists, so the init steps are skipped (LocalStack is provisioned either
    /// way; its state isn't volume-backed).
    FromSnapshot,
}

/// Bring up the backend infra the app services connect to at startup, and get it
/// fully ready before any of them start. The clean-slate teardown means
/// everything is freshly created each run, so this ordering is what stops the
/// services racing their backends (missing `macrodb`, DynamoDB/OpenSearch
/// refused). Dev points Postgres/OpenSearch/FusionAuth at shared dev resources,
/// so it only runs Redis + LocalStack locally.
fn bring_up_infra(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
    init: InfraInit,
) -> Result<()> {
    if stage.is_dry_run() {
        return Ok(());
    }
    let spec = mode.spec();

    // `--wait` gates on each service's healthcheck. Postgres (`pg_isready`) and
    // OpenSearch (HTTP) have patient ones, so this blocks until they genuinely
    // accept connections — not just an open port (Docker's proxy accepts before
    // the server is ready, which is why a plain TCP probe let sqlx hit a
    // not-ready Postgres). Redis + LocalStack have no healthcheck, so `--wait`
    // only confirms "running"; LocalStack readiness is polled below. FusionAuth
    // is deliberately excluded — its healthcheck gives up (5 retries) long before
    // its ~minute kickstart finishes, so it's started + polled separately.
    let waited: &[&str] = if spec.runs_local_infra {
        &["postgres", "redis", "search", "kafka", "localstack"]
    } else {
        &["redis", "kafka", "localstack"]
    };
    let mut up = compose_cmd(instance, env);
    up.arg("up").arg("-d").arg("--wait").args(waited);
    stage.run("Starting infra (docker compose up -d --wait)", &mut up)?;

    // LocalStack has no healthcheck: poll its API, then provision S3/SQS/DynamoDB.
    let localstack_health = format!(
        "http://localhost:{}/_localstack/health",
        instance.port(Port::LocalStack)
    );
    wait_http(stage, "Waiting for LocalStack", &localstack_health)?;
    stage.run_step("Provisioning LocalStack", || {
        localstack::provision(instance)
    })?;

    if spec.migrates_db && init == InfraInit::Full {
        // Postgres is ready (`--wait`); create + migrate the freshly-wiped DB.
        db::migrate(stage, instance)?;
    }
    if spec.runs_local_infra {
        // Kafka is healthy (`--wait` gates on its broker healthcheck); create
        // the event topics declared in `macro_event_topics` — the local
        // equivalent of the MSK topic provisioning driven by the generated
        // `.github/kafka-cluster-topics.json`. Restored volumes already carry
        // the topics (they live in the broker's data dir), so only a full init
        // provisions — which also means a snapshot-restoring `stack up`
        // never needs the rdkafka-backed `local-stack` feature.
        if init == InfraInit::Full {
            stage.run_step("Creating Kafka topics", || kafka::provision(instance))?;
        }

        // Start FusionAuth on its own (impatient healthcheck → no `--wait`) and
        // poll it patiently until it's up. On a full init that wait covers the
        // ~minute kickstart; on a snapshot restore the kickstart is already in
        // the restored volumes, so this is just JVM boot.
        let mut fa = compose_cmd(instance, env);
        fa.arg("up").arg("-d").arg("fusionauth");
        stage.run("Starting FusionAuth", &mut fa)?;
        fusionauth::wait_ready(stage, instance)?;

        // OpenSearch is up (`--wait`) but empty. Create the search indices +
        // aliases (idempotent) so the unified search path works out of the box
        // instead of 404ing on the missing `documents`/`chats`/… indices.
        // Restored volumes already contain them.
        if init == InfraInit::Full {
            opensearch::provision_indices(stage, instance, &env.merged)?;
        }
    }
    Ok(())
}

/// Bring up the app (Rust) services once the infra is ready. For local, `up`
/// with no service args starts everything — the app services plus the auxiliary
/// containers (sync/websocket/lexical/mailpit/proxy/…) — with the already-running
/// infra a no-op, so inter-service start order is unchanged from before. Dev
/// starts only the binaries + proxy (its local infra is already up).
fn bring_up_app(
    stage: &Stage,
    mode: Mode,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
) -> Result<()> {
    let mut up = compose_cmd(instance, env);
    up.arg("up").arg("-d").arg("--remove-orphans");
    if !mode.spec().runs_local_infra {
        for svc in inventory::services_for_mode(mode) {
            up.arg(svc.compose_name);
        }
        up.arg("proxy");
    }
    stage.run("Starting services (docker compose up -d)", &mut up)?;
    connect_tracing_network(instance);
    Ok(())
}

/// Start the requested trace collector (`--traces`) under its own compose
/// project, the same one a developer would use manually — see
/// `docker/docker-compose.yml`'s `jaeger`/`datadog-agent` services. Global and
/// idempotent (like `start_localstack`): one collector per machine, shared
/// across instances, left running across `run_local` invocations.
fn ensure_tracing_backend(stage: &Stage, backend: cli::TracesBackend) -> Result<()> {
    let (Some(profile), Some(service)) = (backend.compose_profile(), backend.compose_service())
    else {
        return Ok(()); // --traces off
    };
    // A keyed backend with a missing key accepts telemetry locally and drops
    // every payload at the vendor intake (403), which looks like "traces are
    // broken" rather than "key is missing" — so fail loud up front.
    if let Some(var) = backend.required_env()
        && macro_env_var::maybe_read_env(var).is_none_or(|v| v.is_empty())
    {
        anyhow::bail!(
            "--traces {profile} requires the {var} env var to be set (export it in \
             the shell you run this from)"
        );
    }
    evict_other_collector(stage, service);
    let result =
        start_global_compose_service(stage, profile, service, "trace collector").and_then(|()| {
            // `up -d` returns once the container starts, not once it's accepting
            // connections; `env_layer::resolve` (which runs right after this, in
            // `prepare`) needs the OTLP port live NOW to decide whether to wire
            // `OTEL_EXPORTER_OTLP_ENDPOINT`, so wait for it here instead of
            // racing. 30s: the LGTM all-in-one is the heaviest collector and
            // takes 10-20s on a cold volume.
            for _ in 0..300 {
                if summary::port_open(4318) {
                    return Ok(());
                }
                std::thread::sleep(std::time::Duration::from_millis(100));
            }
            anyhow::bail!("{profile} trace collector did not come up on port 4318 in time")
        });
    // The default backend must not brick a run that never asked for tracing
    // (offline, image pull failure): warn loud and continue without export —
    // the env probe simply won't wire `OTEL_EXPORTER_OTLP_ENDPOINT`. An
    // explicitly keyed backend still fails hard above.
    match result {
        Err(e) if backend == cli::TracesBackend::Lgtm => {
            stage.note(&format!(
                "  WARNING: could not start the lgtm trace collector ({e:#}); \
                 continuing without tracing (`--traces off` silences this)"
            ));
            Ok(())
        }
        other => other,
    }
}

/// Remove a *different* collector holding the OTLP ports before starting the
/// requested one. All backends bind 4317/4318, so a leftover collector from a
/// previous `--traces` choice would otherwise win the port race and keep
/// receiving the telemetry — silently, since the port probe only asks "is
/// something listening", not "is it the backend that was asked for".
fn evict_other_collector(stage: &Stage, wanted_service: &str) {
    if !summary::port_open(4318) {
        return;
    }
    let Some(holder) = Command::new("docker")
        .args(["ps", "--filter", "publish=4318", "--format", "{{.Names}}"])
        .output()
        .ok()
        .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())
        .filter(|name| !name.is_empty())
    else {
        return; // not a docker container (host collector) — leave it alone
    };
    // Compose names containers `<project>-<service>-<n>`.
    if holder.contains(&format!("-{wanted_service}-")) {
        return;
    }
    stage.note(&format!(
        "  replacing running trace collector {holder} (all collectors share the OTLP ports)"
    ));
    let _ = Command::new("docker").args(["rm", "-f", &holder]).output();
}

/// Start a profile-gated service from the base compose file under the global
/// `macro` project (one per machine, shared across instances, left running
/// across invocations) — the pattern the trace collectors and headless Chrome
/// share. Keep other containers in the shared project: the base compose file
/// does not include generated infrastructure services such as LocalStack.
fn start_global_compose_service(
    stage: &Stage,
    profile: &str,
    service: &str,
    label: &str,
) -> Result<()> {
    let compose = repo_root().join("docker/docker-compose.yml");
    let mut up = Command::new("docker");
    up.arg("compose")
        .arg("--project-directory")
        .arg(repo_root())
        .arg("-f")
        .arg(&compose)
        .arg("--profile")
        .arg(profile)
        .arg("up")
        .arg("-d")
        .arg(service);
    stage.run(&format!("Starting {profile} {label}"), &mut up)
}

/// Start the shared headless Chrome (compose profile `chrome`, CDP on 9222)
/// that agents drive over the DevTools protocol — see the `headless-chrome`
/// service in `docker/docker-compose.yml` and the `live-debug` skill.
/// Best-effort: a machine that can't pull the image still gets a full stack.
fn ensure_headless_chrome(stage: &Stage) {
    if summary::port_open(9222) {
        return; // already running (ours or a host Chrome — either works)
    }
    if let Err(e) = start_global_compose_service(stage, "chrome", "headless-chrome", "browser") {
        stage.note(&format!(
            "  WARNING: could not start headless Chrome ({e:#}); browser \
             debugging via CDP on :9222 will be unavailable"
        ));
    }
}

/// If the global trace collector (Jaeger or the Datadog agent) is running,
/// attach it to this instance's `services` network under the `otel-collector`
/// alias that the injected `OTEL_EXPORTER_OTLP_ENDPOINT` points at (see
/// `env_layer::resolve`). The collectors are started under their own compose
/// project (`docker/docker-compose.yml`, profiles `jaeger`/`datadog`), and
/// Compose prefixes networks per project — so without this, instance
/// containers can't resolve `otel-collector` and span exports fail with DNS
/// errors. Best-effort: "already connected" (rerun) and other failures are
/// ignored, they only mean traces don't flow.
fn connect_tracing_network(instance: &Instance) {
    if !summary::port_open(4318) {
        return;
    }
    // Find the collector container (Jaeger or Datadog agent — both publish the
    // OTLP HTTP port) by that port rather than assuming a container name, in
    // case it was started under a different project.
    let Some(collector) = Command::new("docker")
        .args(["ps", "--filter", "publish=4318", "--format", "{{.Names}}"])
        .output()
        .ok()
        .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())
        .filter(|name| !name.is_empty())
    else {
        return;
    };
    let _ = Command::new("docker")
        .args(["network", "connect", "--alias", "otel-collector"])
        .arg(format!("{}_services", instance.project_name()))
        .arg(collector)
        .output();
}

/// Undo [`connect_tracing_network`] before teardown. The collector lives in a
/// different compose project, and Docker refuses to remove a network with a
/// foreign container still attached — without this, every `compose down`
/// leaves the instance's `services` network behind ("active endpoints",
/// silently swallowed). Best-effort like its counterpart.
fn disconnect_tracing_network(instance: &Instance) {
    let network = format!("{}_services", instance.project_name());
    let Some(attached) = Command::new("docker")
        .args([
            "network",
            "inspect",
            &network,
            "--format",
            "{{range .Containers}}{{.Name}}\n{{end}}",
        ])
        .output()
        .ok()
        .filter(|out| out.status.success())
        .map(|out| String::from_utf8_lossy(&out.stdout).into_owned())
    else {
        return;
    };
    // Foreign means "not this compose project". Decide by the compose project
    // label, not the container name: the global collector `macro-lgtm-1`
    // (project `macro`, service `lgtm`) is name-indistinguishable from a
    // container of an instance literally named `lgtm` (project `macro-lgtm`).
    // One `docker inspect` for all attached containers, not one per name.
    let names: Vec<&str> = attached
        .lines()
        .map(str::trim)
        .filter(|n| !n.is_empty())
        .collect();
    if names.is_empty() {
        return;
    }
    let Some(labeled) = Command::new("docker")
        .args(["inspect", "--format"])
        .arg(r#"{{.Name}} {{index .Config.Labels "com.docker.compose.project"}}"#)
        .args(&names)
        .output()
        .ok()
        .map(|out| String::from_utf8_lossy(&out.stdout).into_owned())
    else {
        return;
    };
    for line in labeled.lines() {
        // `{{.Name}}` carries a leading slash.
        let Some((name, project)) = line.split_once(' ') else {
            continue;
        };
        if project != instance.project_name() {
            let _ = Command::new("docker")
                .args(["network", "disconnect", "-f", &network])
                .arg(name.trim_start_matches('/'))
                .output();
        }
    }
}

/// Recreate the given services' containers so they re-exec their freshly built
/// binaries (bind-mounted at `/app/out`) under the current env file. No
/// graceful-stop grace, and only the changed containers, so a reload after a
/// small change bounces one or two services rather than all twelve. Touching
/// only the changed subset also avoids the `depends_on` race a full restart
/// hits (a service coming back before a peer it depends on).
fn reload_services(
    stage: &Stage,
    instance: &Instance,
    env: &env_layer::ResolvedEnv,
    services: &[&inventory::RustService],
) -> Result<()> {
    if stage.is_dry_run() {
        return Ok(());
    }
    // Recreate rather than `docker restart`: a restart keeps the env the
    // container was created with, so a build that needs a newly required var
    // would crash-loop even though the regenerated env file carries it.
    let mut up = compose_cmd(instance, env);
    up.args([
        "up",
        "-d",
        "--force-recreate",
        "--no-deps",
        "--timeout",
        "0",
    ]);
    for svc in services {
        up.arg(svc.compose_name);
    }
    let names = services
        .iter()
        .map(|s| s.compose_name)
        .collect::<Vec<_>>()
        .join(", ");
    stage.run(&format!("Reloading {names}"), &mut up)
}

/// Every Docker volume an instance owns: the app DB/cache/search/Kafka plus the
/// FusionAuth DB/config. The single list `teardown`, `ensure_external_resources`,
/// and `destroy` share.
fn instance_volumes(instance: &Instance) -> [String; 6] {
    [
        instance.volume_postgres(),
        instance.volume_redis(),
        instance.volume_opensearch(),
        instance.volume_kafka(),
        instance.volume_fusionauth_db(),
        instance.volume_fusionauth_config(),
    ]
}

/// The instance's external Docker networks.
fn instance_networks(instance: &Instance) -> [String; 2] {
    [instance.network_databases(), instance.network_auth()]
}

/// The docker teardown work: stop + remove the instance's stack and ALL its
/// stateful volumes — the basis of `run_local`'s full-delete/full-create
/// idempotency, the quit hotkey, and `destroy-local`. Containers + project
/// networks come down with `down -v`; the data volumes are then removed
/// explicitly, because FusionAuth's (and named instances' infra) volumes are
/// declared `external`, which `down -v` leaves behind. Output is captured (this
/// is best-effort and noisy) so callers surface it as a single line; absent
/// containers/volumes are ignored.
fn teardown_commands(instance: &Instance) {
    sdk_webhook::stop(instance);
    // Detach the global trace collector first or `down` can't remove the
    // instance's `services` network (foreign container = active endpoint).
    disconnect_tracing_network(instance);
    let project = instance.project_name();
    // `-t 0`: SIGKILL immediately, no graceful-shutdown grace. The default 10s
    // SIGTERM timeout per container (Postgres' smart shutdown, OpenSearch, …)
    // dominates teardown — and we're wiping the volumes anyway, so a clean
    // shutdown buys nothing.
    let _ = Command::new("docker")
        .args([
            "compose",
            "-p",
            project,
            "down",
            "-v",
            "--remove-orphans",
            "-t",
            "0",
        ])
        .output();
    for vol in instance_volumes(instance) {
        let _ = Command::new("docker")
            .args(["volume", "rm", "-f", &vol])
            .output();
    }
}

/// [`teardown_commands`] shown as a single progress row (for the quit hotkey and
/// `destroy-local`). `run_local` instead runs it in the background — see
/// `run_stack`.
fn teardown(stage: &Stage, instance: &Instance) -> Result<()> {
    if stage.is_dry_run() {
        return Ok(());
    }
    stage.run_step("Tearing down stack", || {
        teardown_commands(instance);
        Ok(())
    })
}

/// Create the per-instance external networks and volumes the compose files
/// reference (idempotent — replaces the old `just create_networks`).
fn ensure_external_resources(stage: &Stage, instance: &Instance) -> Result<()> {
    let networks = instance_networks(instance);
    let volumes = instance_volumes(instance);
    stage.run_step("Ensuring networks & volumes", || {
        for n in &networks {
            docker::ensure_network(n)?;
        }
        for v in &volumes {
            docker::ensure_volume(v)?;
        }
        Ok(())
    })
}

fn wait_http(stage: &Stage, label: &str, url: &str) -> Result<()> {
    let ca = proxy::curl_ca_args(url);
    let ca_flags = ca
        .iter()
        .map(|arg| format!("'{arg}'"))
        .collect::<Vec<_>>()
        .join(" ");
    let script = format!(
        "for i in $(seq 1 600); do curl -fsS {ca_flags} --max-time 3 {url} >/dev/null 2>&1 && exit 0; sleep 0.2; done; echo 'not ready: {url}'; exit 1"
    );
    let mut cmd = Command::new("bash");
    cmd.arg("-lc").arg(script);
    stage.run(label, &mut cmd)
}

/// `cargo x zigbuild`.
pub fn zigbuild_only() -> Result<()> {
    let stage = Stage::from_env();
    let target = arch::detect()?;
    let binaries = build::resolve(
        &stage,
        target,
        &build::BuildOptions {
            no_build: false,
            binaries_dir: None,
        },
    )?;
    stage.note(&format!("binaries at {}", binaries.host_dir().display()));
    Ok(())
}

/// `cargo x runtime-image`.
pub fn runtime_image_only(force: bool) -> Result<()> {
    let stage = Stage::from_env();
    build::ensure_runtime_image(&stage, arch::detect()?, force)
}

/// `cargo x gen-compose`.
pub fn gen_compose_only(args: &cli::InstanceArgs) -> Result<()> {
    let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;
    let target = arch::detect()?;
    let binaries = build::BinariesDir::TargetDir(workspace_root().join(target.debug_dir()));
    let path = gen_compose::generate(Mode::Local, &instance, &binaries, false, false)?;
    println!("{}", path.display());
    Ok(())
}

/// `cargo x stop-local` — stop the instance's containers (keep volumes).
pub fn stop(args: &cli::InstanceArgs) -> Result<()> {
    let stage = Stage::from_env();
    let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;
    sdk_webhook::stop(&instance);
    let mut cmd = Command::new("docker");
    cmd.args(["compose", "-p", instance.project_name(), "stop"]);
    stage.run(&format!("Stopping {}", instance.project_name()), &mut cmd)
}

/// `cargo x reset-local` — drop, recreate, and migrate the instance DB.
pub fn reset(args: &cli::InstanceArgs) -> Result<()> {
    let stage = Stage::from_env();
    let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;
    db::reset(&stage, &instance)
}

/// `cargo x destroy-local` — remove the instance's containers, volumes, and
/// (named instances) external networks.
pub fn destroy(args: &cli::InstanceArgs) -> Result<()> {
    let stage = Stage::from_env();
    let instance = Instance::derive(args.instance.as_deref(), args.port_base)?;
    // Containers + all volumes (incl. FusionAuth's external ones, which `down -v`
    // leaves behind — the same teardown run_local does for its clean slate).
    teardown(&stage, &instance)?;
    // The per-instance external networks only exist for named instances; the
    // default instance's are shared base-compose networks we leave in place.
    if !instance.is_default() {
        stage.run_step("Removing instance networks", || {
            for n in instance_networks(&instance) {
                let _ = Command::new("docker").args(["network", "rm", &n]).status();
            }
            Ok(())
        })?;
    }
    Ok(())
}